cloud-mu 2.0.0.pre.alpha9 → 2.0.0.pre.beta1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Berksfile.lock +1 -1
- data/README.md +2 -0
- data/bin/mu-configure +2 -58
- data/bin/mu-gen-docs +29 -4
- data/bin/mu-load-config.rb +0 -1
- data/bin/mu-user-manage +4 -0
- data/cloud-mu.gemspec +2 -2
- data/cookbooks/mu-master/recipes/default.rb +3 -4
- data/cookbooks/mu-master/recipes/init.rb +3 -3
- data/cookbooks/mu-tools/files/default/Mu_CA.pem +15 -15
- data/cookbooks/mu-tools/libraries/helper.rb +1 -1
- data/cookbooks/mu-tools/recipes/eks.rb +3 -3
- data/cookbooks/mu-tools/recipes/set_local_fw.rb +1 -1
- data/cookbooks/mu-utility/recipes/remi.rb +1 -1
- data/cookbooks/nagios/libraries/base.rb +4 -4
- data/cookbooks/nagios/libraries/contact.rb +1 -1
- data/cookbooks/nagios/libraries/contactgroup.rb +1 -1
- data/cookbooks/nagios/libraries/host.rb +2 -2
- data/cookbooks/nagios/libraries/hostdependency.rb +3 -3
- data/cookbooks/nagios/libraries/hostescalation.rb +3 -3
- data/cookbooks/nagios/libraries/hostgroup.rb +2 -2
- data/cookbooks/nagios/libraries/nagios.rb +5 -5
- data/cookbooks/nagios/libraries/service.rb +3 -3
- data/cookbooks/nagios/libraries/servicedependency.rb +2 -2
- data/cookbooks/nagios/libraries/serviceescalation.rb +2 -2
- data/cookbooks/nagios/libraries/servicegroup.rb +2 -2
- data/cookbooks/nagios/libraries/timeperiod.rb +1 -1
- data/install/installer +1 -1
- data/modules/mu/cleanup.rb +1 -1
- data/modules/mu/cloud.rb +43 -1
- data/modules/mu/clouds/aws.rb +55 -35
- data/modules/mu/clouds/aws/bucket.rb +287 -0
- data/modules/mu/clouds/aws/database.rb +65 -11
- data/modules/mu/clouds/aws/endpoint.rb +592 -0
- data/modules/mu/clouds/aws/firewall_rule.rb +4 -0
- data/modules/mu/clouds/aws/function.rb +138 -93
- data/modules/mu/clouds/aws/nosqldb.rb +387 -0
- data/modules/mu/clouds/aws/role.rb +1 -1
- data/modules/mu/clouds/aws/server.rb +5 -5
- data/modules/mu/clouds/aws/server_pool.rb +60 -3
- data/modules/mu/clouds/azure.rb +0 -1
- data/modules/mu/clouds/google.rb +34 -12
- data/modules/mu/clouds/google/bucket.rb +179 -0
- data/modules/mu/config.rb +1 -1
- data/modules/mu/config/bucket.rb +69 -0
- data/modules/mu/config/bucket.yml +10 -0
- data/modules/mu/config/database.rb +1 -1
- data/modules/mu/config/endpoint.rb +71 -0
- data/modules/mu/config/function.rb +6 -0
- data/modules/mu/config/nosqldb.rb +49 -0
- data/modules/mu/config/nosqldb.yml +44 -0
- data/modules/mu/config/notifier.yml +2 -2
- data/modules/mu/config/vpc.rb +0 -1
- data/modules/mu/defaults/amazon_images.yaml +32 -30
- data/modules/mu/groomers/chef.rb +1 -1
- data/modules/mu/kittens.rb +2430 -1511
- data/modules/mu/master/ldap.rb +1 -1
- data/modules/tests/super_complex_bok.yml +7 -0
- data/modules/tests/super_simple_bok.yml +7 -0
- metadata +11 -2
@@ -248,6 +248,10 @@ module MU
|
|
248
248
|
tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
|
249
249
|
end
|
250
250
|
|
251
|
+
# Some services create sneaky rogue ENIs which then block removal of
|
252
|
+
# associated security groups. Find them and fry them.
|
253
|
+
MU::Cloud::AWS::VPC.purge_interfaces(noop, tagfilters, region: region, credentials: credentials)
|
254
|
+
|
251
255
|
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_security_groups(
|
252
256
|
filters: tagfilters
|
253
257
|
)
|
@@ -35,69 +35,7 @@ module MU
|
|
35
35
|
@mu_name ||= @deploy.getResourceName(@config["name"])
|
36
36
|
end
|
37
37
|
|
38
|
-
#
|
39
|
-
# sibling Mu role resources by this name first, and failing that, will
|
40
|
-
# do a plain get_role() to the IAM API for the provided name.
|
41
|
-
# @param name [String]
|
42
|
-
def get_role_arn(name)
|
43
|
-
sib_role = @deploy.findLitterMate(name: name, type: "roles")
|
44
|
-
return sib_role.cloudobj.arn if sib_role
|
45
|
-
|
46
|
-
begin
|
47
|
-
role = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role({
|
48
|
-
role_name: name.to_s
|
49
|
-
})
|
50
|
-
return role['role']['arn']
|
51
|
-
rescue Exception => e
|
52
|
-
MU.log "#{e}", MU::ERR
|
53
|
-
end
|
54
|
-
nil
|
55
|
-
end
|
56
|
-
|
57
|
-
def get_vpc_config(vpc_name, subnet_name, sg_name,region=@config['region'])
|
58
|
-
if !subnet_name.nil? and !sg_name.nil? and !vpc_name.nil?
|
59
|
-
## get vpc_id
|
60
|
-
## get sub_id and verify its in the same vpc
|
61
|
-
## get sg_id and verify its in the same vpc
|
62
|
-
ec2_client = MU::Cloud::AWS.ec2(region: region, credentials: @config['credentials'])
|
63
|
-
|
64
|
-
vpc_filter = ec2_client.describe_vpcs({
|
65
|
-
filters: [{ name: 'tag-value', values: [vpc_name] }]
|
66
|
-
})
|
67
|
-
bok_vpc_id = vpc_filter.vpcs[0].vpc_id
|
68
|
-
|
69
|
-
sub_filter = ec2_client.describe_subnets({
|
70
|
-
filters: [{ name: 'tag-value', values: [subnet_name] }]
|
71
|
-
})
|
72
|
-
|
73
|
-
sub_id = nil
|
74
|
-
sub_filter.subnets.each do |each|
|
75
|
-
if each.vpc_id == bok_vpc_id
|
76
|
-
sub_id = each.subnet_id
|
77
|
-
break
|
78
|
-
end
|
79
|
-
end
|
80
|
-
|
81
|
-
sg_filter = ec2_client.describe_security_groups({
|
82
|
-
filters: [{ name: 'group-name', values: [sg_name] }]
|
83
|
-
})
|
84
|
-
|
85
|
-
|
86
|
-
if sg_filter.security_groups[0].vpc_id.to_s != bok_vpc_id
|
87
|
-
MU.log "Security Group: #{sg_name} is not part of the VPC: #{vpc_name}", MU::ERR
|
88
|
-
raise MuError, "Please provide security group name that exists in the vpc"
|
89
|
-
end
|
90
|
-
|
91
|
-
#sub_id = sub_filter.subnets[0].subnet_id
|
92
|
-
sg_id = sg_filter.security_groups[0].group_id
|
93
|
-
return {subnet_ids: [sub_id], security_group_ids: [sg_id]}
|
94
|
-
else
|
95
|
-
MU.log "Function: #{@config['name']}, Missing either subnet_name or security_group_name or vpc_name in the vpc stanza!", MU::ERR
|
96
|
-
raise MuError, "Insufficient parameters for locating vpc resource ids ==> #{@config['name']}"
|
97
|
-
end
|
98
|
-
end
|
99
|
-
|
100
|
-
|
38
|
+
# Tag this Lambda function
|
101
39
|
def assign_tag(resource_arn, tag_list, region=@config['region'])
|
102
40
|
begin
|
103
41
|
tag_list.each do |each_pair|
|
@@ -162,17 +100,21 @@ module MU
|
|
162
100
|
end
|
163
101
|
|
164
102
|
if @config.has_key?('vpc')
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
103
|
+
sgs = []
|
104
|
+
if @config['add_firewall_rules']
|
105
|
+
@config['add_firewall_rules'].each { |sg|
|
106
|
+
sg = @deploy.findLitterMate(type: "firewall_rule", name: sg['rule_name'])
|
107
|
+
sgs << sg.cloud_id if sg and sg.cloud_id
|
108
|
+
}
|
109
|
+
end
|
110
|
+
lambda_properties[:vpc_config] = {
|
111
|
+
:subnet_ids => @config['vpc']['subnets'].map { |s| s["subnet_id"] },
|
112
|
+
:security_group_ids => sgs
|
113
|
+
}
|
172
114
|
end
|
173
115
|
|
174
116
|
retries = 0
|
175
|
-
begin
|
117
|
+
resp = begin
|
176
118
|
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties)
|
177
119
|
rescue Aws::Lambda::Errors::InvalidParameterValueException => e
|
178
120
|
# Freshly-made IAM roles sometimes aren't really ready
|
@@ -183,8 +125,12 @@ module MU
|
|
183
125
|
end
|
184
126
|
raise e
|
185
127
|
end
|
128
|
+
|
129
|
+
|
130
|
+
@cloud_id = resp.function_name
|
186
131
|
end
|
187
132
|
|
133
|
+
# Called automatically by {MU::Deploy#createResources}
|
188
134
|
def groom
|
189
135
|
desc = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).get_function(
|
190
136
|
function_name: @mu_name
|
@@ -212,14 +158,11 @@ module MU
|
|
212
158
|
source_arn: trigger_arn,
|
213
159
|
statement_id: "#{@mu_name}-ID-1",
|
214
160
|
}
|
215
|
-
p trigger_arn
|
216
|
-
p trigger_properties
|
217
161
|
|
218
162
|
MU.log trigger_properties, MU::DEBUG
|
219
163
|
begin
|
220
164
|
add_trigger = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties)
|
221
165
|
rescue Aws::Lambda::Errors::ResourceConflictException
|
222
|
-
# XXX check properly for existence
|
223
166
|
end
|
224
167
|
adjust_trigger(tr['service'], trigger_arn, func_arn, @mu_name)
|
225
168
|
}
|
@@ -227,7 +170,36 @@ module MU
|
|
227
170
|
end
|
228
171
|
end
|
229
172
|
|
173
|
+
# Intended to be called by other Mu resources, such as Endpoints (API
|
174
|
+
# Gateways) to add themselves as triggers for this Lambda function.
|
175
|
+
def addTrigger(calling_arn, calling_service, calling_name)
|
176
|
+
trigger = {
|
177
|
+
action: "lambda:InvokeFunction",
|
178
|
+
function_name: @mu_name,
|
179
|
+
principal: "#{calling_service}.amazonaws.com",
|
180
|
+
source_arn: calling_arn,
|
181
|
+
statement_id: "#{calling_service}-#{calling_name}",
|
182
|
+
}
|
230
183
|
|
184
|
+
begin
|
185
|
+
# XXX There doesn't seem to be an API call to list or view existing
|
186
|
+
# permissions, wtaf. This means we can't intelligently guard this.
|
187
|
+
add_trigger = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger)
|
188
|
+
rescue Aws::Lambda::Errors::ResourceConflictException => e
|
189
|
+
if e.message.match(/already exists/)
|
190
|
+
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).remove_permission(
|
191
|
+
function_name: @mu_name,
|
192
|
+
statement_id: "#{calling_service}-#{calling_name}"
|
193
|
+
)
|
194
|
+
retry
|
195
|
+
else
|
196
|
+
MU.log "Error trying to add trigger to Lambda #{@mu_name}: #{e.message}", MU::ERR, details: trigger
|
197
|
+
raise e
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
# Look up an ARN for a given trigger type and resource name
|
231
203
|
def assume_trigger_arns(svc, name)
|
232
204
|
supported_triggers = %w(apigateway sns events event cloudwatch_event)
|
233
205
|
if supported_triggers.include?(svc.downcase)
|
@@ -249,7 +221,7 @@ module MU
|
|
249
221
|
return arn
|
250
222
|
end
|
251
223
|
|
252
|
-
|
224
|
+
# XXX placeholder, really; this is going end up being done from Endpoint, Log and Notification resources, I think
|
253
225
|
def adjust_trigger(trig_type, trig_arn, func_arn, func_id=nil, protocol='lambda',region=@config['region'])
|
254
226
|
|
255
227
|
case trig_type
|
@@ -274,7 +246,8 @@ module MU
|
|
274
246
|
]
|
275
247
|
})
|
276
248
|
when 'apigateway'
|
277
|
-
|
249
|
+
# XXX this is actually happening in ::Endpoint... maybe...
|
250
|
+
# MU.log "Creation of API Gateway integrations not yet implemented, you'll have to do this manually", MU::WARN, details: "(because we'll basically have to implement all of APIG for this)"
|
278
251
|
end
|
279
252
|
end
|
280
253
|
|
@@ -282,8 +255,8 @@ module MU
|
|
282
255
|
# Return the metadata for this Function rule
|
283
256
|
# @return [Hash]
|
284
257
|
def notify
|
285
|
-
deploy_struct =
|
286
|
-
|
258
|
+
deploy_struct = MU.structToHash(MU::Cloud::AWS::Function.find(cloud_id: @cloud_id, credentials: @config['credentials'], region: @config['region']).values.first)
|
259
|
+
deploy_struct['mu_name'] = @mu_name
|
287
260
|
return deploy_struct
|
288
261
|
end
|
289
262
|
|
@@ -327,21 +300,20 @@ module MU
|
|
327
300
|
# @param region [String]: The cloud provider region.
|
328
301
|
# @param flags [Hash]: Optional flags
|
329
302
|
# @return [OpenStruct]: The cloud provider's complete descriptions of matching function.
|
330
|
-
def self.find(cloud_id: nil,
|
331
|
-
|
332
|
-
|
303
|
+
def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
|
304
|
+
matches = {}
|
305
|
+
|
306
|
+
if !cloud_id.nil?
|
333
307
|
all_functions = MU::Cloud::AWS.lambda(region: region, credentials: credentials).list_functions
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
break
|
339
|
-
end
|
308
|
+
all_functions.functions.each do |x|
|
309
|
+
if x.function_name == cloud_id
|
310
|
+
matches[x.function_name] = x
|
311
|
+
break
|
340
312
|
end
|
341
313
|
end
|
342
314
|
end
|
343
315
|
|
344
|
-
return
|
316
|
+
return matches
|
345
317
|
end
|
346
318
|
|
347
319
|
|
@@ -355,8 +327,17 @@ module MU
|
|
355
327
|
schema = {
|
356
328
|
"iam_role" => {
|
357
329
|
"type" => "string",
|
358
|
-
"description" => "The name of an IAM role for our Lambda function to assume. Can refer to an existing IAM role, or a sibling 'role' resource in Mu. If not specified, will create a default role with
|
359
|
-
}
|
330
|
+
"description" => "The name of an IAM role for our Lambda function to assume. Can refer to an existing IAM role, or a sibling 'role' resource in Mu. If not specified, will create a default role with permissions listed in `permissions` (and if none are listed, we will set `AWSLambdaBasicExecutionRole`)."
|
331
|
+
},
|
332
|
+
"permissions" => {
|
333
|
+
"type" => "array",
|
334
|
+
"description" => "if `iam_role` is unspecified, we will create a default execution role for our function, and add one or more permissions to it.",
|
335
|
+
"items" => {
|
336
|
+
"type" => "string",
|
337
|
+
"description" => "A permission to add to our Lambda function's default role, corresponding to standard AWS policies (see https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html)",
|
338
|
+
"enum" => ["basic", "kinesis", "dynamo", "sqs", "network", "xray"]
|
339
|
+
}
|
340
|
+
},
|
360
341
|
# XXX add some canned permission sets here, asking people to get the AWS weirdness right and then translate it into Mu-speak is just too much. Think about auto-populating when a target log group is asked for, mappings for the AWS canned policies in the URL above, writes to arbitrary S3 buckets, etc
|
361
342
|
}
|
362
343
|
[toplevel_required, schema]
|
@@ -369,7 +350,52 @@ module MU
|
|
369
350
|
def self.validateConfig(function, configurator)
|
370
351
|
ok = true
|
371
352
|
|
353
|
+
if function['vpc']
|
354
|
+
fwname = "lambda-#{function['name']}"
|
355
|
+
# default to allowing pings, if no ingress_rules were specified
|
356
|
+
function['ingress_rules'] ||= [
|
357
|
+
{
|
358
|
+
"proto" => "icmp",
|
359
|
+
"hosts" => ["0.0.0.0/0"]
|
360
|
+
}
|
361
|
+
]
|
362
|
+
acl = {
|
363
|
+
"name" => fwname,
|
364
|
+
"rules" => function['ingress_rules'],
|
365
|
+
"region" => function['region'],
|
366
|
+
"credentials" => function['credentials'],
|
367
|
+
"optional_tags" => function['optional_tags']
|
368
|
+
}
|
369
|
+
acl["tags"] = function['tags'] if function['tags'] && !function['tags'].empty?
|
370
|
+
acl["vpc"] = function['vpc'].dup if function['vpc']
|
371
|
+
ok = false if !configurator.insertKitten(acl, "firewall_rules")
|
372
|
+
function["add_firewall_rules"] = [] if function["add_firewall_rules"].nil?
|
373
|
+
function["add_firewall_rules"] << {"rule_name" => fwname}
|
374
|
+
function["permissions"] ||= []
|
375
|
+
function["permissions"] << "network"
|
376
|
+
function['dependencies'] ||= []
|
377
|
+
function['dependencies'] << {
|
378
|
+
"name" => fwname,
|
379
|
+
"type" => "firewall_rule"
|
380
|
+
}
|
381
|
+
end
|
382
|
+
|
372
383
|
if !function['iam_role']
|
384
|
+
policy_map = {
|
385
|
+
"basic" => "AWSLambdaBasicExecutionRole",
|
386
|
+
"kinesis" => "AWSLambdaKinesisExecutionRole",
|
387
|
+
"dynamo" => "AWSLambdaDynamoDBExecutionRole",
|
388
|
+
"sqs" => "AWSLambdaSQSQueueExecutionRole ",
|
389
|
+
"network" => "AWSLambdaVPCAccessExecutionRole",
|
390
|
+
"xray" => "AWSXrayWriteOnlyAccess"
|
391
|
+
}
|
392
|
+
policies = if function['permissions']
|
393
|
+
function['permissions'].map { |p|
|
394
|
+
policy_map[p]
|
395
|
+
}
|
396
|
+
else
|
397
|
+
["AWSLambdaBasicExecutionRole"]
|
398
|
+
end
|
373
399
|
roledesc = {
|
374
400
|
"name" => function['name']+"execrole",
|
375
401
|
"credentials" => function['credentials'],
|
@@ -379,9 +405,7 @@ module MU
|
|
379
405
|
"entity_type" => "service"
|
380
406
|
}
|
381
407
|
],
|
382
|
-
"import" =>
|
383
|
-
"AWSLambdaBasicExecutionRole"
|
384
|
-
]
|
408
|
+
"import" => policies
|
385
409
|
}
|
386
410
|
configurator.insertKitten(roledesc, "roles")
|
387
411
|
|
@@ -397,6 +421,27 @@ module MU
|
|
397
421
|
ok
|
398
422
|
end
|
399
423
|
|
424
|
+
private
|
425
|
+
|
426
|
+
# Given an IAM role name, resolve to ARN. Will attempt to identify any
|
427
|
+
# sibling Mu role resources by this name first, and failing that, will
|
428
|
+
# do a plain get_role() to the IAM API for the provided name.
|
429
|
+
# @param name [String]
|
430
|
+
def get_role_arn(name)
|
431
|
+
sib_role = @deploy.findLitterMate(name: name, type: "roles")
|
432
|
+
return sib_role.cloudobj.arn if sib_role
|
433
|
+
|
434
|
+
begin
|
435
|
+
role = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role({
|
436
|
+
role_name: name.to_s
|
437
|
+
})
|
438
|
+
return role['role']['arn']
|
439
|
+
rescue Exception => e
|
440
|
+
MU.log "#{e}", MU::ERR
|
441
|
+
end
|
442
|
+
nil
|
443
|
+
end
|
444
|
+
|
400
445
|
end
|
401
446
|
end
|
402
447
|
end
|
@@ -0,0 +1,387 @@
|
|
1
|
+
# Copyright:: Copyright (c) 2019 eGlobalTech, Inc., all rights reserved
|
2
|
+
#
|
3
|
+
# Licensed under the BSD-3 license (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License in the root of the project or at
|
6
|
+
#
|
7
|
+
# http://egt-labs.com/mu/LICENSE.html
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
module MU
|
16
|
+
class Cloud
|
17
|
+
class AWS
|
18
|
+
# Support for AWS DynamoDB
|
19
|
+
class NoSQLDB < MU::Cloud::NoSQLDB
|
20
|
+
@deploy = nil
|
21
|
+
@config = nil
|
22
|
+
|
23
|
+
@@region_cache = {}
|
24
|
+
@@region_cache_semaphore = Mutex.new
|
25
|
+
|
26
|
+
attr_reader :mu_name
|
27
|
+
attr_reader :config
|
28
|
+
attr_reader :cloud_id
|
29
|
+
|
30
|
+
# @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
|
31
|
+
# @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::logs}
|
32
|
+
def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
|
33
|
+
@deploy = mommacat
|
34
|
+
@config = MU::Config.manxify(kitten_cfg)
|
35
|
+
@cloud_id ||= cloud_id
|
36
|
+
@mu_name ||= @deploy.getResourceName(@config["name"])
|
37
|
+
end
|
38
|
+
|
39
|
+
# Called automatically by {MU::Deploy#createResources}
|
40
|
+
def create
|
41
|
+
params = {
|
42
|
+
:table_name => @mu_name,
|
43
|
+
:attribute_definitions => [],
|
44
|
+
:key_schema => [],
|
45
|
+
:provisioned_throughput => {
|
46
|
+
:read_capacity_units => @config['read_capacity'],
|
47
|
+
:write_capacity_units => @config['write_capacity']
|
48
|
+
}
|
49
|
+
}
|
50
|
+
|
51
|
+
if @config['stream']
|
52
|
+
params[:stream_specification] = {
|
53
|
+
:stream_enabled => true,
|
54
|
+
:stream_view_type => @config['stream']
|
55
|
+
}
|
56
|
+
end
|
57
|
+
|
58
|
+
@config['attributes'].each { |attr|
|
59
|
+
params[:attribute_definitions] << {
|
60
|
+
:attribute_name => attr['name'],
|
61
|
+
:attribute_type => attr['type']
|
62
|
+
}
|
63
|
+
|
64
|
+
if attr['primary_partition']
|
65
|
+
params[:key_schema] << {
|
66
|
+
:attribute_name => attr['name'],
|
67
|
+
:key_type => "HASH"
|
68
|
+
}
|
69
|
+
end
|
70
|
+
|
71
|
+
if attr['primary_sort']
|
72
|
+
params[:key_schema] << {
|
73
|
+
:attribute_name => attr['name'],
|
74
|
+
:key_type => "RANGE"
|
75
|
+
}
|
76
|
+
end
|
77
|
+
}
|
78
|
+
|
79
|
+
if @config['secondary_indexes']
|
80
|
+
@config['secondary_indexes'].each { |idx|
|
81
|
+
idx_cfg = {
|
82
|
+
:index_name => idx['index_name'],
|
83
|
+
:projection => {
|
84
|
+
:projection_type => idx['projection']['type'],
|
85
|
+
:non_key_attributes => idx['projection']['non_key_attributes']
|
86
|
+
},
|
87
|
+
:key_schema => []
|
88
|
+
}
|
89
|
+
idx['key_schema'].each { |attr|
|
90
|
+
idx_cfg[:key_schema] << {
|
91
|
+
:attribute_name => attr['attribute'],
|
92
|
+
:key_type => attr['type']
|
93
|
+
}
|
94
|
+
}
|
95
|
+
if idx['type'] == "global"
|
96
|
+
|
97
|
+
idx_cfg[:provisioned_throughput] = {
|
98
|
+
:read_capacity_units => idx['read_capacity'],
|
99
|
+
:write_capacity_units => idx['write_capacity']
|
100
|
+
}
|
101
|
+
params[:global_secondary_indexes] ||= []
|
102
|
+
params[:global_secondary_indexes] << idx_cfg
|
103
|
+
else
|
104
|
+
params[:local_secondary_indexes] ||= []
|
105
|
+
params[:local_secondary_indexes] << idx_cfg
|
106
|
+
end
|
107
|
+
}
|
108
|
+
end
|
109
|
+
pp params
|
110
|
+
MU.log "Creating DynamoDB table #{@mu_name}", details: params
|
111
|
+
|
112
|
+
resp = MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).create_table(params)
|
113
|
+
@cloud_id = @mu_name
|
114
|
+
|
115
|
+
begin
|
116
|
+
resp = MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).describe_table(table_name: @cloud_id)
|
117
|
+
sleep 5 if resp.table.table_status == "CREATING"
|
118
|
+
end while resp.table.table_status == "CREATING"
|
119
|
+
|
120
|
+
|
121
|
+
tagTable if !@config['scrub_mu_isms']
|
122
|
+
end
|
123
|
+
|
124
|
+
# Apply tags to this DynamoDB table
|
125
|
+
def tagTable
|
126
|
+
tagset = []
|
127
|
+
|
128
|
+
MU::MommaCat.listStandardTags.each_pair { |key, value|
|
129
|
+
tagset << { :key => key, :value => value }
|
130
|
+
}
|
131
|
+
|
132
|
+
if @config['tags']
|
133
|
+
@config['tags'].each { |tag|
|
134
|
+
tagset << { :key => tag['key'], :value => tag['value'] }
|
135
|
+
}
|
136
|
+
end
|
137
|
+
|
138
|
+
if @config['optional_tags']
|
139
|
+
MU::MommaCat.listOptionalTags.each { |key, value|
|
140
|
+
tagset << { :key => key, :value => value }
|
141
|
+
}
|
142
|
+
end
|
143
|
+
|
144
|
+
MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).tag_resource(
|
145
|
+
resource_arn: arn,
|
146
|
+
tags: tagset
|
147
|
+
)
|
148
|
+
|
149
|
+
end
|
150
|
+
|
151
|
+
# Called automatically by {MU::Deploy#createResources}
|
152
|
+
def groom
|
153
|
+
tagTable if !@config['scrub_mu_isms']
|
154
|
+
end
|
155
|
+
|
156
|
+
# Does this resource type exist as a global (cloud-wide) artifact, or
|
157
|
+
# is it localized to a region/zone?
|
158
|
+
# @return [Boolean]
|
159
|
+
def self.isGlobal?
|
160
|
+
false
|
161
|
+
end
|
162
|
+
|
163
|
+
# Remove all buckets associated with the currently loaded deployment.
|
164
|
+
# @param noop [Boolean]: If true, will only print what would be done
|
165
|
+
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
166
|
+
# @param region [String]: The cloud provider region
|
167
|
+
# @return [void]
|
168
|
+
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
169
|
+
resp = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).list_tables
|
170
|
+
if resp and resp.table_names
|
171
|
+
resp.table_names.each { |table|
|
172
|
+
desc = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).describe_table(table_name: table).table
|
173
|
+
next if desc.table_status == "DELETING"
|
174
|
+
tags = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).list_tags_of_resource(resource_arn: desc.table_arn)
|
175
|
+
if tags and tags.tags
|
176
|
+
tags.tags.each { |tag|
|
177
|
+
if tag.key == "MU-ID" and tag.value == MU.deploy_id
|
178
|
+
MU.log "Deleting DynamoDB table #{desc.table_name}"
|
179
|
+
if !noop
|
180
|
+
MU::Cloud::AWS.dynamo(credentials: credentials, region: region).delete_table(table_name: desc.table_name)
|
181
|
+
end
|
182
|
+
end
|
183
|
+
}
|
184
|
+
end
|
185
|
+
|
186
|
+
}
|
187
|
+
end
|
188
|
+
|
189
|
+
end
|
190
|
+
|
191
|
+
# Canonical Amazon Resource Number for this resource
|
192
|
+
# @return [String]
|
193
|
+
def arn
|
194
|
+
return nil if cloud_desc.nil?
|
195
|
+
cloud_desc.table_arn
|
196
|
+
end
|
197
|
+
|
198
|
+
# Return the metadata for this user cofiguration
|
199
|
+
# @return [Hash]
|
200
|
+
def notify
|
201
|
+
MU.structToHash(cloud_desc)
|
202
|
+
end
|
203
|
+
|
204
|
+
# Locate an existing DynamoDB table
|
205
|
+
# @param cloud_id [String]: The cloud provider's identifier for this resource.
|
206
|
+
# @param region [String]: The cloud provider region.
|
207
|
+
# @param flags [Hash]: Optional flags
|
208
|
+
# @return [OpenStruct]: The cloud provider's complete descriptions of matching bucket.
|
209
|
+
def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
|
210
|
+
found = {}
|
211
|
+
if cloud_id
|
212
|
+
resp = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).describe_table(table_name: cloud_id)
|
213
|
+
found[cloud_id] = resp.table if resp and resp.table
|
214
|
+
end
|
215
|
+
found
|
216
|
+
end
|
217
|
+
|
218
|
+
# Cloud-specific configuration properties.
|
219
|
+
# @param config [MU::Config]: The calling MU::Config object
|
220
|
+
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
221
|
+
def self.schema(config)
|
222
|
+
toplevel_required = ["attributes"]
|
223
|
+
|
224
|
+
|
225
|
+
schema = {
|
226
|
+
"attributes" => {
|
227
|
+
"type" => "array",
|
228
|
+
"minItems" => 1,
|
229
|
+
"items" => {
|
230
|
+
"type" => "object",
|
231
|
+
"description" => "Fields for data we'll be storing in this database, somewhat akin to SQL columns. Note that all attributes declared here must be a +primary_partition+, +primary_sort+, or named in a +secondary_index+.",
|
232
|
+
"properties" => {
|
233
|
+
"name" => {
|
234
|
+
"type" => "string",
|
235
|
+
"description" => "The name of this attribute"
|
236
|
+
},
|
237
|
+
"type" => {
|
238
|
+
"type" => "string",
|
239
|
+
"description" => "The type of attribute; S = String, N = Number, B = Binary",
|
240
|
+
"enum" => ["S", "N", "B"]
|
241
|
+
},
|
242
|
+
"primary_partition" => {
|
243
|
+
"type" => "boolean",
|
244
|
+
"default" => false
|
245
|
+
},
|
246
|
+
"primary_sort" => {
|
247
|
+
"type" => "boolean",
|
248
|
+
"default" => false
|
249
|
+
}
|
250
|
+
}
|
251
|
+
}
|
252
|
+
},
|
253
|
+
"read_capacity" => {
|
254
|
+
"type" => "integer",
|
255
|
+
"description" => "Provisioned read throughput. See also: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ProvisionedThroughput.html",
|
256
|
+
"default" => 1
|
257
|
+
},
|
258
|
+
"write_capacity" => {
|
259
|
+
"type" => "integer",
|
260
|
+
"description" => "Provisioned write throughput. See also: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ProvisionedThroughput.html",
|
261
|
+
"default" => 1
|
262
|
+
},
|
263
|
+
"stream" => {
|
264
|
+
"type" => "string",
|
265
|
+
"description" => "If specified, enables a streaming log of changes to this DynamoDB table. See also https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Streams.html",
|
266
|
+
"enum" => ["NEW_IMAGE", "OLD_IMAGE", "NEW_AND_OLD_IMAGES", "KEYS_ONLY"]
|
267
|
+
},
|
268
|
+
"secondary_indexes" => {
|
269
|
+
"type" => "array",
|
270
|
+
"description" => "Define a global and/or a local secondary index.",
|
271
|
+
"items" => {
|
272
|
+
"type" => "object",
|
273
|
+
"description" => "An index with a partition key and a sort key that can be different from those on the base table; queries on the index can span all of the data in the base table, across all partitions",
|
274
|
+
"required" => ["index_name", "key_schema", "projection"],
|
275
|
+
"properties" => {
|
276
|
+
"index_name" => {
|
277
|
+
"type" => "string",
|
278
|
+
"description" => "A name for this index"
|
279
|
+
},
|
280
|
+
"type" => {
|
281
|
+
"type" => "string",
|
282
|
+
"description" => "Whether to create a global or local secondary index. See also: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/SecondaryIndexes.html",
|
283
|
+
"enum" => ["global", "local"],
|
284
|
+
"default" => "global"
|
285
|
+
},
|
286
|
+
"projection" => {
|
287
|
+
"type" => "object",
|
288
|
+
"description" => "The set of attributes to return for queries against this index.",
|
289
|
+
"properties" => {
|
290
|
+
"type" => {
|
291
|
+
"type" => "string",
|
292
|
+
"enum" => ["ALL", "KEYS_ONLY", "INCLUDE"],
|
293
|
+
"default" => "ALL"
|
294
|
+
},
|
295
|
+
"non_key_attributes" => {
|
296
|
+
"type" => "array",
|
297
|
+
"items" => {
|
298
|
+
"type" => "string",
|
299
|
+
"description" => "The name of an extra attribute to include in results for queries against this index"
|
300
|
+
}
|
301
|
+
}
|
302
|
+
},
|
303
|
+
"default" => { "type" => "ALL" }
|
304
|
+
},
|
305
|
+
"read_capacity" => {
|
306
|
+
"type" => "integer",
|
307
|
+
"description" => "Provisioned read throughput. Only valid for global secondary indexes. Defaults to the read capacity of the whole table.",
|
308
|
+
},
|
309
|
+
"write_capacity" => {
|
310
|
+
"type" => "integer",
|
311
|
+
"description" => "Provisioned write throughput. Only valid for global secondary indexes. Defaults to the read capacity of the whole table.",
|
312
|
+
},
|
313
|
+
"key_schema" => {
|
314
|
+
"type" => "array",
|
315
|
+
"minItems" => 1,
|
316
|
+
"items" => {
|
317
|
+
"type" => "object",
|
318
|
+
"description" => "Define the key for this index, which most be composed of one or more declared +attributes+ for this table. See also: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/SecondaryIndexes.html",
|
319
|
+
"properties" => {
|
320
|
+
"type" => {
|
321
|
+
"type" => "string",
|
322
|
+
"enum" => ["HASH", "RANGE"]
|
323
|
+
},
|
324
|
+
"attribute" => {
|
325
|
+
"description" => "This must refer to a declared +attribute+ by name",
|
326
|
+
"type" => "string",
|
327
|
+
}
|
328
|
+
|
329
|
+
}
|
330
|
+
}
|
331
|
+
}
|
332
|
+
}
|
333
|
+
|
334
|
+
}
|
335
|
+
}
|
336
|
+
}
|
337
|
+
[toplevel_required, schema]
|
338
|
+
end
|
339
|
+
|
340
|
+
# Cloud-specific pre-processing of {MU::Config::BasketofKittens::nosqldbs}, bare and unvalidated.
|
341
|
+
|
342
|
+
# @param db [Hash]: The resource to process and validate
|
343
|
+
# @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
|
344
|
+
# @return [Boolean]: True if validation succeeded, False otherwise
|
345
|
+
def self.validateConfig(db, configurator)
|
346
|
+
ok = true
|
347
|
+
|
348
|
+
partition = nil
|
349
|
+
db['attributes'].each { |attr|
|
350
|
+
if attr['primary_partition']
|
351
|
+
partition = attr['name']
|
352
|
+
end
|
353
|
+
}
|
354
|
+
if !partition
|
355
|
+
if db['attributes'].size == 1
|
356
|
+
MU.log "NoSQL database '#{db['name']}' only declares one attribute; setting '#{db['attributes'].first['name']}' as primary partition key", MU::NOTICE
|
357
|
+
db['attributes'].first['primary_partition'] = true
|
358
|
+
else
|
359
|
+
MU.log "NoSQL database '#{db['name']}' must have an attribute flagged as primary_partition", MU::ERR
|
360
|
+
ok = false
|
361
|
+
end
|
362
|
+
end
|
363
|
+
db['attributes'].each { |attr|
|
364
|
+
if attr['primary_partition'] and attr['primary_sort']
|
365
|
+
MU.log "NoSQL database '#{db['name']}' attribute '#{attr['name']}' cannot be both primary_partition and primary_sort", MU::ERR
|
366
|
+
ok = false
|
367
|
+
end
|
368
|
+
}
|
369
|
+
|
370
|
+
if db['secondary_indexes']
|
371
|
+
db['secondary_indexes'].each { |idx|
|
372
|
+
if idx['type'] == "global"
|
373
|
+
idx['read_capacity'] ||= db['read_capacity']
|
374
|
+
idx['write_capacity'] ||= db['write_capacity']
|
375
|
+
end
|
376
|
+
}
|
377
|
+
end
|
378
|
+
|
379
|
+
ok
|
380
|
+
end
|
381
|
+
|
382
|
+
private
|
383
|
+
|
384
|
+
end
|
385
|
+
end
|
386
|
+
end
|
387
|
+
end
|