cloud-mu 2.0.0.pre.alpha9 → 2.0.0.pre.beta1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/Berksfile.lock +1 -1
  3. data/README.md +2 -0
  4. data/bin/mu-configure +2 -58
  5. data/bin/mu-gen-docs +29 -4
  6. data/bin/mu-load-config.rb +0 -1
  7. data/bin/mu-user-manage +4 -0
  8. data/cloud-mu.gemspec +2 -2
  9. data/cookbooks/mu-master/recipes/default.rb +3 -4
  10. data/cookbooks/mu-master/recipes/init.rb +3 -3
  11. data/cookbooks/mu-tools/files/default/Mu_CA.pem +15 -15
  12. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  13. data/cookbooks/mu-tools/recipes/eks.rb +3 -3
  14. data/cookbooks/mu-tools/recipes/set_local_fw.rb +1 -1
  15. data/cookbooks/mu-utility/recipes/remi.rb +1 -1
  16. data/cookbooks/nagios/libraries/base.rb +4 -4
  17. data/cookbooks/nagios/libraries/contact.rb +1 -1
  18. data/cookbooks/nagios/libraries/contactgroup.rb +1 -1
  19. data/cookbooks/nagios/libraries/host.rb +2 -2
  20. data/cookbooks/nagios/libraries/hostdependency.rb +3 -3
  21. data/cookbooks/nagios/libraries/hostescalation.rb +3 -3
  22. data/cookbooks/nagios/libraries/hostgroup.rb +2 -2
  23. data/cookbooks/nagios/libraries/nagios.rb +5 -5
  24. data/cookbooks/nagios/libraries/service.rb +3 -3
  25. data/cookbooks/nagios/libraries/servicedependency.rb +2 -2
  26. data/cookbooks/nagios/libraries/serviceescalation.rb +2 -2
  27. data/cookbooks/nagios/libraries/servicegroup.rb +2 -2
  28. data/cookbooks/nagios/libraries/timeperiod.rb +1 -1
  29. data/install/installer +1 -1
  30. data/modules/mu/cleanup.rb +1 -1
  31. data/modules/mu/cloud.rb +43 -1
  32. data/modules/mu/clouds/aws.rb +55 -35
  33. data/modules/mu/clouds/aws/bucket.rb +287 -0
  34. data/modules/mu/clouds/aws/database.rb +65 -11
  35. data/modules/mu/clouds/aws/endpoint.rb +592 -0
  36. data/modules/mu/clouds/aws/firewall_rule.rb +4 -0
  37. data/modules/mu/clouds/aws/function.rb +138 -93
  38. data/modules/mu/clouds/aws/nosqldb.rb +387 -0
  39. data/modules/mu/clouds/aws/role.rb +1 -1
  40. data/modules/mu/clouds/aws/server.rb +5 -5
  41. data/modules/mu/clouds/aws/server_pool.rb +60 -3
  42. data/modules/mu/clouds/azure.rb +0 -1
  43. data/modules/mu/clouds/google.rb +34 -12
  44. data/modules/mu/clouds/google/bucket.rb +179 -0
  45. data/modules/mu/config.rb +1 -1
  46. data/modules/mu/config/bucket.rb +69 -0
  47. data/modules/mu/config/bucket.yml +10 -0
  48. data/modules/mu/config/database.rb +1 -1
  49. data/modules/mu/config/endpoint.rb +71 -0
  50. data/modules/mu/config/function.rb +6 -0
  51. data/modules/mu/config/nosqldb.rb +49 -0
  52. data/modules/mu/config/nosqldb.yml +44 -0
  53. data/modules/mu/config/notifier.yml +2 -2
  54. data/modules/mu/config/vpc.rb +0 -1
  55. data/modules/mu/defaults/amazon_images.yaml +32 -30
  56. data/modules/mu/groomers/chef.rb +1 -1
  57. data/modules/mu/kittens.rb +2430 -1511
  58. data/modules/mu/master/ldap.rb +1 -1
  59. data/modules/tests/super_complex_bok.yml +7 -0
  60. data/modules/tests/super_simple_bok.yml +7 -0
  61. metadata +11 -2
@@ -248,6 +248,10 @@ module MU
248
248
  tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
249
249
  end
250
250
 
251
+ # Some services create sneaky rogue ENIs which then block removal of
252
+ # associated security groups. Find them and fry them.
253
+ MU::Cloud::AWS::VPC.purge_interfaces(noop, tagfilters, region: region, credentials: credentials)
254
+
251
255
  resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_security_groups(
252
256
  filters: tagfilters
253
257
  )
@@ -35,69 +35,7 @@ module MU
35
35
  @mu_name ||= @deploy.getResourceName(@config["name"])
36
36
  end
37
37
 
38
- # Given an IAM role name, resolve to ARN. Will attempt to identify any
39
- # sibling Mu role resources by this name first, and failing that, will
40
- # do a plain get_role() to the IAM API for the provided name.
41
- # @param name [String]
42
- def get_role_arn(name)
43
- sib_role = @deploy.findLitterMate(name: name, type: "roles")
44
- return sib_role.cloudobj.arn if sib_role
45
-
46
- begin
47
- role = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role({
48
- role_name: name.to_s
49
- })
50
- return role['role']['arn']
51
- rescue Exception => e
52
- MU.log "#{e}", MU::ERR
53
- end
54
- nil
55
- end
56
-
57
- def get_vpc_config(vpc_name, subnet_name, sg_name,region=@config['region'])
58
- if !subnet_name.nil? and !sg_name.nil? and !vpc_name.nil?
59
- ## get vpc_id
60
- ## get sub_id and verify its in the same vpc
61
- ## get sg_id and verify its in the same vpc
62
- ec2_client = MU::Cloud::AWS.ec2(region: region, credentials: @config['credentials'])
63
-
64
- vpc_filter = ec2_client.describe_vpcs({
65
- filters: [{ name: 'tag-value', values: [vpc_name] }]
66
- })
67
- bok_vpc_id = vpc_filter.vpcs[0].vpc_id
68
-
69
- sub_filter = ec2_client.describe_subnets({
70
- filters: [{ name: 'tag-value', values: [subnet_name] }]
71
- })
72
-
73
- sub_id = nil
74
- sub_filter.subnets.each do |each|
75
- if each.vpc_id == bok_vpc_id
76
- sub_id = each.subnet_id
77
- break
78
- end
79
- end
80
-
81
- sg_filter = ec2_client.describe_security_groups({
82
- filters: [{ name: 'group-name', values: [sg_name] }]
83
- })
84
-
85
-
86
- if sg_filter.security_groups[0].vpc_id.to_s != bok_vpc_id
87
- MU.log "Security Group: #{sg_name} is not part of the VPC: #{vpc_name}", MU::ERR
88
- raise MuError, "Please provide security group name that exists in the vpc"
89
- end
90
-
91
- #sub_id = sub_filter.subnets[0].subnet_id
92
- sg_id = sg_filter.security_groups[0].group_id
93
- return {subnet_ids: [sub_id], security_group_ids: [sg_id]}
94
- else
95
- MU.log "Function: #{@config['name']}, Missing either subnet_name or security_group_name or vpc_name in the vpc stanza!", MU::ERR
96
- raise MuError, "Insufficient parameters for locating vpc resource ids ==> #{@config['name']}"
97
- end
98
- end
99
-
100
-
38
+ # Tag this Lambda function
101
39
  def assign_tag(resource_arn, tag_list, region=@config['region'])
102
40
  begin
103
41
  tag_list.each do |each_pair|
@@ -162,17 +100,21 @@ module MU
162
100
  end
163
101
 
164
102
  if @config.has_key?('vpc')
165
- ### get vpc and subnet_name
166
- ### find the subnet_id
167
- sub_name = @config['vpc']['subnet_name']
168
- vpc_name = @config['vpc']['vpc_name']
169
- sg_name = @config['vpc']['security_group_name']
170
- vpc_conf = get_vpc_config(vpc_name,sub_name,sg_name)
171
- lambda_properties[:vpc_config] = vpc_conf
103
+ sgs = []
104
+ if @config['add_firewall_rules']
105
+ @config['add_firewall_rules'].each { |sg|
106
+ sg = @deploy.findLitterMate(type: "firewall_rule", name: sg['rule_name'])
107
+ sgs << sg.cloud_id if sg and sg.cloud_id
108
+ }
109
+ end
110
+ lambda_properties[:vpc_config] = {
111
+ :subnet_ids => @config['vpc']['subnets'].map { |s| s["subnet_id"] },
112
+ :security_group_ids => sgs
113
+ }
172
114
  end
173
115
 
174
116
  retries = 0
175
- begin
117
+ resp = begin
176
118
  MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties)
177
119
  rescue Aws::Lambda::Errors::InvalidParameterValueException => e
178
120
  # Freshly-made IAM roles sometimes aren't really ready
@@ -183,8 +125,12 @@ module MU
183
125
  end
184
126
  raise e
185
127
  end
128
+
129
+
130
+ @cloud_id = resp.function_name
186
131
  end
187
132
 
133
+ # Called automatically by {MU::Deploy#createResources}
188
134
  def groom
189
135
  desc = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).get_function(
190
136
  function_name: @mu_name
@@ -212,14 +158,11 @@ module MU
212
158
  source_arn: trigger_arn,
213
159
  statement_id: "#{@mu_name}-ID-1",
214
160
  }
215
- p trigger_arn
216
- p trigger_properties
217
161
 
218
162
  MU.log trigger_properties, MU::DEBUG
219
163
  begin
220
164
  add_trigger = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties)
221
165
  rescue Aws::Lambda::Errors::ResourceConflictException
222
- # XXX check properly for existence
223
166
  end
224
167
  adjust_trigger(tr['service'], trigger_arn, func_arn, @mu_name)
225
168
  }
@@ -227,7 +170,36 @@ module MU
227
170
  end
228
171
  end
229
172
 
173
+ # Intended to be called by other Mu resources, such as Endpoints (API
174
+ # Gateways) to add themselves as triggers for this Lambda function.
175
+ def addTrigger(calling_arn, calling_service, calling_name)
176
+ trigger = {
177
+ action: "lambda:InvokeFunction",
178
+ function_name: @mu_name,
179
+ principal: "#{calling_service}.amazonaws.com",
180
+ source_arn: calling_arn,
181
+ statement_id: "#{calling_service}-#{calling_name}",
182
+ }
230
183
 
184
+ begin
185
+ # XXX There doesn't seem to be an API call to list or view existing
186
+ # permissions, wtaf. This means we can't intelligently guard this.
187
+ add_trigger = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger)
188
+ rescue Aws::Lambda::Errors::ResourceConflictException => e
189
+ if e.message.match(/already exists/)
190
+ MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).remove_permission(
191
+ function_name: @mu_name,
192
+ statement_id: "#{calling_service}-#{calling_name}"
193
+ )
194
+ retry
195
+ else
196
+ MU.log "Error trying to add trigger to Lambda #{@mu_name}: #{e.message}", MU::ERR, details: trigger
197
+ raise e
198
+ end
199
+ end
200
+ end
201
+
202
+ # Look up an ARN for a given trigger type and resource name
231
203
  def assume_trigger_arns(svc, name)
232
204
  supported_triggers = %w(apigateway sns events event cloudwatch_event)
233
205
  if supported_triggers.include?(svc.downcase)
@@ -249,7 +221,7 @@ module MU
249
221
  return arn
250
222
  end
251
223
 
252
-
224
+ # XXX placeholder, really; this is going end up being done from Endpoint, Log and Notification resources, I think
253
225
  def adjust_trigger(trig_type, trig_arn, func_arn, func_id=nil, protocol='lambda',region=@config['region'])
254
226
 
255
227
  case trig_type
@@ -274,7 +246,8 @@ module MU
274
246
  ]
275
247
  })
276
248
  when 'apigateway'
277
- MU.log "Creation of API Gateway integrations not yet implemented, you'll have to do this manually", MU::WARN, details: "(because we'll basically have to implement all of APIG for this)"
249
+ # XXX this is actually happening in ::Endpoint... maybe...
250
+ # MU.log "Creation of API Gateway integrations not yet implemented, you'll have to do this manually", MU::WARN, details: "(because we'll basically have to implement all of APIG for this)"
278
251
  end
279
252
  end
280
253
 
@@ -282,8 +255,8 @@ module MU
282
255
  # Return the metadata for this Function rule
283
256
  # @return [Hash]
284
257
  def notify
285
- deploy_struct = {
286
- }
258
+ deploy_struct = MU.structToHash(MU::Cloud::AWS::Function.find(cloud_id: @cloud_id, credentials: @config['credentials'], region: @config['region']).values.first)
259
+ deploy_struct['mu_name'] = @mu_name
287
260
  return deploy_struct
288
261
  end
289
262
 
@@ -327,21 +300,20 @@ module MU
327
300
  # @param region [String]: The cloud provider region.
328
301
  # @param flags [Hash]: Optional flags
329
302
  # @return [OpenStruct]: The cloud provider's complete descriptions of matching function.
330
- def self.find(cloud_id: nil, func_name: nil, region: MU.curRegion, credentials: nil, flags: {})
331
- func = nil
332
- if !func_name.nil?
303
+ def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
304
+ matches = {}
305
+
306
+ if !cloud_id.nil?
333
307
  all_functions = MU::Cloud::AWS.lambda(region: region, credentials: credentials).list_functions
334
- if all_functions.include?(func_name)
335
- all_functions.functions.each do |x|
336
- if x.function_name == func_name
337
- func = x
338
- break
339
- end
308
+ all_functions.functions.each do |x|
309
+ if x.function_name == cloud_id
310
+ matches[x.function_name] = x
311
+ break
340
312
  end
341
313
  end
342
314
  end
343
315
 
344
- return func
316
+ return matches
345
317
  end
346
318
 
347
319
 
@@ -355,8 +327,17 @@ module MU
355
327
  schema = {
356
328
  "iam_role" => {
357
329
  "type" => "string",
358
- "description" => "The name of an IAM role for our Lambda function to assume. Can refer to an existing IAM role, or a sibling 'role' resource in Mu. If not specified, will create a default role with the AWSLambdaBasicExecutionRole policy attached. To grant other permissions for your function, create a Mu 'role' resource and use the 'import' and 'policies' parameters to add permissions. See also: https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html"
359
- }
330
+ "description" => "The name of an IAM role for our Lambda function to assume. Can refer to an existing IAM role, or a sibling 'role' resource in Mu. If not specified, will create a default role with permissions listed in `permissions` (and if none are listed, we will set `AWSLambdaBasicExecutionRole`)."
331
+ },
332
+ "permissions" => {
333
+ "type" => "array",
334
+ "description" => "if `iam_role` is unspecified, we will create a default execution role for our function, and add one or more permissions to it.",
335
+ "items" => {
336
+ "type" => "string",
337
+ "description" => "A permission to add to our Lambda function's default role, corresponding to standard AWS policies (see https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html)",
338
+ "enum" => ["basic", "kinesis", "dynamo", "sqs", "network", "xray"]
339
+ }
340
+ },
360
341
  # XXX add some canned permission sets here, asking people to get the AWS weirdness right and then translate it into Mu-speak is just too much. Think about auto-populating when a target log group is asked for, mappings for the AWS canned policies in the URL above, writes to arbitrary S3 buckets, etc
361
342
  }
362
343
  [toplevel_required, schema]
@@ -369,7 +350,52 @@ module MU
369
350
  def self.validateConfig(function, configurator)
370
351
  ok = true
371
352
 
353
+ if function['vpc']
354
+ fwname = "lambda-#{function['name']}"
355
+ # default to allowing pings, if no ingress_rules were specified
356
+ function['ingress_rules'] ||= [
357
+ {
358
+ "proto" => "icmp",
359
+ "hosts" => ["0.0.0.0/0"]
360
+ }
361
+ ]
362
+ acl = {
363
+ "name" => fwname,
364
+ "rules" => function['ingress_rules'],
365
+ "region" => function['region'],
366
+ "credentials" => function['credentials'],
367
+ "optional_tags" => function['optional_tags']
368
+ }
369
+ acl["tags"] = function['tags'] if function['tags'] && !function['tags'].empty?
370
+ acl["vpc"] = function['vpc'].dup if function['vpc']
371
+ ok = false if !configurator.insertKitten(acl, "firewall_rules")
372
+ function["add_firewall_rules"] = [] if function["add_firewall_rules"].nil?
373
+ function["add_firewall_rules"] << {"rule_name" => fwname}
374
+ function["permissions"] ||= []
375
+ function["permissions"] << "network"
376
+ function['dependencies'] ||= []
377
+ function['dependencies'] << {
378
+ "name" => fwname,
379
+ "type" => "firewall_rule"
380
+ }
381
+ end
382
+
372
383
  if !function['iam_role']
384
+ policy_map = {
385
+ "basic" => "AWSLambdaBasicExecutionRole",
386
+ "kinesis" => "AWSLambdaKinesisExecutionRole",
387
+ "dynamo" => "AWSLambdaDynamoDBExecutionRole",
388
+ "sqs" => "AWSLambdaSQSQueueExecutionRole ",
389
+ "network" => "AWSLambdaVPCAccessExecutionRole",
390
+ "xray" => "AWSXrayWriteOnlyAccess"
391
+ }
392
+ policies = if function['permissions']
393
+ function['permissions'].map { |p|
394
+ policy_map[p]
395
+ }
396
+ else
397
+ ["AWSLambdaBasicExecutionRole"]
398
+ end
373
399
  roledesc = {
374
400
  "name" => function['name']+"execrole",
375
401
  "credentials" => function['credentials'],
@@ -379,9 +405,7 @@ module MU
379
405
  "entity_type" => "service"
380
406
  }
381
407
  ],
382
- "import" => [
383
- "AWSLambdaBasicExecutionRole"
384
- ]
408
+ "import" => policies
385
409
  }
386
410
  configurator.insertKitten(roledesc, "roles")
387
411
 
@@ -397,6 +421,27 @@ module MU
397
421
  ok
398
422
  end
399
423
 
424
+ private
425
+
426
+ # Given an IAM role name, resolve to ARN. Will attempt to identify any
427
+ # sibling Mu role resources by this name first, and failing that, will
428
+ # do a plain get_role() to the IAM API for the provided name.
429
+ # @param name [String]
430
+ def get_role_arn(name)
431
+ sib_role = @deploy.findLitterMate(name: name, type: "roles")
432
+ return sib_role.cloudobj.arn if sib_role
433
+
434
+ begin
435
+ role = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role({
436
+ role_name: name.to_s
437
+ })
438
+ return role['role']['arn']
439
+ rescue Exception => e
440
+ MU.log "#{e}", MU::ERR
441
+ end
442
+ nil
443
+ end
444
+
400
445
  end
401
446
  end
402
447
  end
@@ -0,0 +1,387 @@
1
+ # Copyright:: Copyright (c) 2019 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ class Cloud
17
+ class AWS
18
+ # Support for AWS DynamoDB
19
+ class NoSQLDB < MU::Cloud::NoSQLDB
20
+ @deploy = nil
21
+ @config = nil
22
+
23
+ @@region_cache = {}
24
+ @@region_cache_semaphore = Mutex.new
25
+
26
+ attr_reader :mu_name
27
+ attr_reader :config
28
+ attr_reader :cloud_id
29
+
30
+ # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
31
+ # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::logs}
32
+ def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
33
+ @deploy = mommacat
34
+ @config = MU::Config.manxify(kitten_cfg)
35
+ @cloud_id ||= cloud_id
36
+ @mu_name ||= @deploy.getResourceName(@config["name"])
37
+ end
38
+
39
+ # Called automatically by {MU::Deploy#createResources}
40
+ def create
41
+ params = {
42
+ :table_name => @mu_name,
43
+ :attribute_definitions => [],
44
+ :key_schema => [],
45
+ :provisioned_throughput => {
46
+ :read_capacity_units => @config['read_capacity'],
47
+ :write_capacity_units => @config['write_capacity']
48
+ }
49
+ }
50
+
51
+ if @config['stream']
52
+ params[:stream_specification] = {
53
+ :stream_enabled => true,
54
+ :stream_view_type => @config['stream']
55
+ }
56
+ end
57
+
58
+ @config['attributes'].each { |attr|
59
+ params[:attribute_definitions] << {
60
+ :attribute_name => attr['name'],
61
+ :attribute_type => attr['type']
62
+ }
63
+
64
+ if attr['primary_partition']
65
+ params[:key_schema] << {
66
+ :attribute_name => attr['name'],
67
+ :key_type => "HASH"
68
+ }
69
+ end
70
+
71
+ if attr['primary_sort']
72
+ params[:key_schema] << {
73
+ :attribute_name => attr['name'],
74
+ :key_type => "RANGE"
75
+ }
76
+ end
77
+ }
78
+
79
+ if @config['secondary_indexes']
80
+ @config['secondary_indexes'].each { |idx|
81
+ idx_cfg = {
82
+ :index_name => idx['index_name'],
83
+ :projection => {
84
+ :projection_type => idx['projection']['type'],
85
+ :non_key_attributes => idx['projection']['non_key_attributes']
86
+ },
87
+ :key_schema => []
88
+ }
89
+ idx['key_schema'].each { |attr|
90
+ idx_cfg[:key_schema] << {
91
+ :attribute_name => attr['attribute'],
92
+ :key_type => attr['type']
93
+ }
94
+ }
95
+ if idx['type'] == "global"
96
+
97
+ idx_cfg[:provisioned_throughput] = {
98
+ :read_capacity_units => idx['read_capacity'],
99
+ :write_capacity_units => idx['write_capacity']
100
+ }
101
+ params[:global_secondary_indexes] ||= []
102
+ params[:global_secondary_indexes] << idx_cfg
103
+ else
104
+ params[:local_secondary_indexes] ||= []
105
+ params[:local_secondary_indexes] << idx_cfg
106
+ end
107
+ }
108
+ end
109
+ pp params
110
+ MU.log "Creating DynamoDB table #{@mu_name}", details: params
111
+
112
+ resp = MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).create_table(params)
113
+ @cloud_id = @mu_name
114
+
115
+ begin
116
+ resp = MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).describe_table(table_name: @cloud_id)
117
+ sleep 5 if resp.table.table_status == "CREATING"
118
+ end while resp.table.table_status == "CREATING"
119
+
120
+
121
+ tagTable if !@config['scrub_mu_isms']
122
+ end
123
+
124
+ # Apply tags to this DynamoDB table
125
+ def tagTable
126
+ tagset = []
127
+
128
+ MU::MommaCat.listStandardTags.each_pair { |key, value|
129
+ tagset << { :key => key, :value => value }
130
+ }
131
+
132
+ if @config['tags']
133
+ @config['tags'].each { |tag|
134
+ tagset << { :key => tag['key'], :value => tag['value'] }
135
+ }
136
+ end
137
+
138
+ if @config['optional_tags']
139
+ MU::MommaCat.listOptionalTags.each { |key, value|
140
+ tagset << { :key => key, :value => value }
141
+ }
142
+ end
143
+
144
+ MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).tag_resource(
145
+ resource_arn: arn,
146
+ tags: tagset
147
+ )
148
+
149
+ end
150
+
151
+ # Called automatically by {MU::Deploy#createResources}
152
+ def groom
153
+ tagTable if !@config['scrub_mu_isms']
154
+ end
155
+
156
+ # Does this resource type exist as a global (cloud-wide) artifact, or
157
+ # is it localized to a region/zone?
158
+ # @return [Boolean]
159
+ def self.isGlobal?
160
+ false
161
+ end
162
+
163
+ # Remove all buckets associated with the currently loaded deployment.
164
+ # @param noop [Boolean]: If true, will only print what would be done
165
+ # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
166
+ # @param region [String]: The cloud provider region
167
+ # @return [void]
168
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
169
+ resp = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).list_tables
170
+ if resp and resp.table_names
171
+ resp.table_names.each { |table|
172
+ desc = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).describe_table(table_name: table).table
173
+ next if desc.table_status == "DELETING"
174
+ tags = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).list_tags_of_resource(resource_arn: desc.table_arn)
175
+ if tags and tags.tags
176
+ tags.tags.each { |tag|
177
+ if tag.key == "MU-ID" and tag.value == MU.deploy_id
178
+ MU.log "Deleting DynamoDB table #{desc.table_name}"
179
+ if !noop
180
+ MU::Cloud::AWS.dynamo(credentials: credentials, region: region).delete_table(table_name: desc.table_name)
181
+ end
182
+ end
183
+ }
184
+ end
185
+
186
+ }
187
+ end
188
+
189
+ end
190
+
191
+ # Canonical Amazon Resource Number for this resource
192
+ # @return [String]
193
+ def arn
194
+ return nil if cloud_desc.nil?
195
+ cloud_desc.table_arn
196
+ end
197
+
198
+ # Return the metadata for this user cofiguration
199
+ # @return [Hash]
200
+ def notify
201
+ MU.structToHash(cloud_desc)
202
+ end
203
+
204
+ # Locate an existing DynamoDB table
205
+ # @param cloud_id [String]: The cloud provider's identifier for this resource.
206
+ # @param region [String]: The cloud provider region.
207
+ # @param flags [Hash]: Optional flags
208
+ # @return [OpenStruct]: The cloud provider's complete descriptions of matching bucket.
209
+ def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
210
+ found = {}
211
+ if cloud_id
212
+ resp = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).describe_table(table_name: cloud_id)
213
+ found[cloud_id] = resp.table if resp and resp.table
214
+ end
215
+ found
216
+ end
217
+
218
+ # Cloud-specific configuration properties.
219
+ # @param config [MU::Config]: The calling MU::Config object
220
+ # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
221
+ def self.schema(config)
222
+ toplevel_required = ["attributes"]
223
+
224
+
225
+ schema = {
226
+ "attributes" => {
227
+ "type" => "array",
228
+ "minItems" => 1,
229
+ "items" => {
230
+ "type" => "object",
231
+ "description" => "Fields for data we'll be storing in this database, somewhat akin to SQL columns. Note that all attributes declared here must be a +primary_partition+, +primary_sort+, or named in a +secondary_index+.",
232
+ "properties" => {
233
+ "name" => {
234
+ "type" => "string",
235
+ "description" => "The name of this attribute"
236
+ },
237
+ "type" => {
238
+ "type" => "string",
239
+ "description" => "The type of attribute; S = String, N = Number, B = Binary",
240
+ "enum" => ["S", "N", "B"]
241
+ },
242
+ "primary_partition" => {
243
+ "type" => "boolean",
244
+ "default" => false
245
+ },
246
+ "primary_sort" => {
247
+ "type" => "boolean",
248
+ "default" => false
249
+ }
250
+ }
251
+ }
252
+ },
253
+ "read_capacity" => {
254
+ "type" => "integer",
255
+ "description" => "Provisioned read throughput. See also: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ProvisionedThroughput.html",
256
+ "default" => 1
257
+ },
258
+ "write_capacity" => {
259
+ "type" => "integer",
260
+ "description" => "Provisioned write throughput. See also: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ProvisionedThroughput.html",
261
+ "default" => 1
262
+ },
263
+ "stream" => {
264
+ "type" => "string",
265
+ "description" => "If specified, enables a streaming log of changes to this DynamoDB table. See also https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Streams.html",
266
+ "enum" => ["NEW_IMAGE", "OLD_IMAGE", "NEW_AND_OLD_IMAGES", "KEYS_ONLY"]
267
+ },
268
+ "secondary_indexes" => {
269
+ "type" => "array",
270
+ "description" => "Define a global and/or a local secondary index.",
271
+ "items" => {
272
+ "type" => "object",
273
+ "description" => "An index with a partition key and a sort key that can be different from those on the base table; queries on the index can span all of the data in the base table, across all partitions",
274
+ "required" => ["index_name", "key_schema", "projection"],
275
+ "properties" => {
276
+ "index_name" => {
277
+ "type" => "string",
278
+ "description" => "A name for this index"
279
+ },
280
+ "type" => {
281
+ "type" => "string",
282
+ "description" => "Whether to create a global or local secondary index. See also: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/SecondaryIndexes.html",
283
+ "enum" => ["global", "local"],
284
+ "default" => "global"
285
+ },
286
+ "projection" => {
287
+ "type" => "object",
288
+ "description" => "The set of attributes to return for queries against this index.",
289
+ "properties" => {
290
+ "type" => {
291
+ "type" => "string",
292
+ "enum" => ["ALL", "KEYS_ONLY", "INCLUDE"],
293
+ "default" => "ALL"
294
+ },
295
+ "non_key_attributes" => {
296
+ "type" => "array",
297
+ "items" => {
298
+ "type" => "string",
299
+ "description" => "The name of an extra attribute to include in results for queries against this index"
300
+ }
301
+ }
302
+ },
303
+ "default" => { "type" => "ALL" }
304
+ },
305
+ "read_capacity" => {
306
+ "type" => "integer",
307
+ "description" => "Provisioned read throughput. Only valid for global secondary indexes. Defaults to the read capacity of the whole table.",
308
+ },
309
+ "write_capacity" => {
310
+ "type" => "integer",
311
+ "description" => "Provisioned write throughput. Only valid for global secondary indexes. Defaults to the read capacity of the whole table.",
312
+ },
313
+ "key_schema" => {
314
+ "type" => "array",
315
+ "minItems" => 1,
316
+ "items" => {
317
+ "type" => "object",
318
+ "description" => "Define the key for this index, which most be composed of one or more declared +attributes+ for this table. See also: https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/SecondaryIndexes.html",
319
+ "properties" => {
320
+ "type" => {
321
+ "type" => "string",
322
+ "enum" => ["HASH", "RANGE"]
323
+ },
324
+ "attribute" => {
325
+ "description" => "This must refer to a declared +attribute+ by name",
326
+ "type" => "string",
327
+ }
328
+
329
+ }
330
+ }
331
+ }
332
+ }
333
+
334
+ }
335
+ }
336
+ }
337
+ [toplevel_required, schema]
338
+ end
339
+
340
+ # Cloud-specific pre-processing of {MU::Config::BasketofKittens::nosqldbs}, bare and unvalidated.
341
+
342
+ # @param db [Hash]: The resource to process and validate
343
+ # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
344
+ # @return [Boolean]: True if validation succeeded, False otherwise
345
+ def self.validateConfig(db, configurator)
346
+ ok = true
347
+
348
+ partition = nil
349
+ db['attributes'].each { |attr|
350
+ if attr['primary_partition']
351
+ partition = attr['name']
352
+ end
353
+ }
354
+ if !partition
355
+ if db['attributes'].size == 1
356
+ MU.log "NoSQL database '#{db['name']}' only declares one attribute; setting '#{db['attributes'].first['name']}' as primary partition key", MU::NOTICE
357
+ db['attributes'].first['primary_partition'] = true
358
+ else
359
+ MU.log "NoSQL database '#{db['name']}' must have an attribute flagged as primary_partition", MU::ERR
360
+ ok = false
361
+ end
362
+ end
363
+ db['attributes'].each { |attr|
364
+ if attr['primary_partition'] and attr['primary_sort']
365
+ MU.log "NoSQL database '#{db['name']}' attribute '#{attr['name']}' cannot be both primary_partition and primary_sort", MU::ERR
366
+ ok = false
367
+ end
368
+ }
369
+
370
+ if db['secondary_indexes']
371
+ db['secondary_indexes'].each { |idx|
372
+ if idx['type'] == "global"
373
+ idx['read_capacity'] ||= db['read_capacity']
374
+ idx['write_capacity'] ||= db['write_capacity']
375
+ end
376
+ }
377
+ end
378
+
379
+ ok
380
+ end
381
+
382
+ private
383
+
384
+ end
385
+ end
386
+ end
387
+ end