elasticDynamoDb 1.4.2 → 1.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGES.md +1 -1
- data/elasticDynamoDb.gemspec +2 -0
- data/lib/elasticDynamoDb.rb +1 -1
- data/lib/elasticDynamoDb/cli.rb +50 -29
- data/lib/elasticDynamoDb/cloudwatch.rb +52 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ab92c209e72914d232932624af454e823ea8916d
|
4
|
+
data.tar.gz: 394e933da2f2318d322d7cc912e46b360624297b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 867c68fc402f5422d86ca890d5a069350f02dbeccc3500792bc7543f13ad4c55162f7b55a42fad436f5d008b643f0500b74dc96873c9bb56a049b6c77f7176de
|
7
|
+
data.tar.gz: 35fa7c1447f695ce8167968599faeebdc854a6078cec488fea26850a138a453d965393b571f280f646daa3fe1bcdd81468e79423d8cd324cdf834a06712b053b
|
data/CHANGES.md
CHANGED
data/elasticDynamoDb.gemspec
CHANGED
data/lib/elasticDynamoDb.rb
CHANGED
data/lib/elasticDynamoDb/cli.rb
CHANGED
@@ -1,15 +1,19 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
require 'thor'
|
3
|
-
require 'aws-sdk
|
3
|
+
require 'aws-sdk'
|
4
4
|
require 'fileutils'
|
5
|
-
|
6
|
-
autoload :ConfigParser, 'elasticDynamoDb/configparser'
|
5
|
+
$:.unshift File.expand_path(File.join(File.dirname(__FILE__), '.'))
|
7
6
|
|
8
7
|
class ElasticDynamoDb::Cli < Thor
|
9
8
|
include Thor::Actions
|
9
|
+
autoload :ConfigParser, 'configparser'
|
10
|
+
autoload :CloudWatch, 'cloudwatch'
|
11
|
+
|
12
|
+
include CloudWatch
|
13
|
+
|
10
14
|
default_task :onDemand
|
11
|
-
attr_accessor :restore_in_progress, :backup_folder, :config_file_name, :original_config_file, :config, :ddb,
|
12
|
-
:log_file, :automated_reason
|
15
|
+
attr_accessor :restore_in_progress, :backup_folder, :config_file_name, :original_config_file, :config, :ddb, :cw,
|
16
|
+
:log_file, :automated_reason, :skip_cloudwatch
|
13
17
|
|
14
18
|
desc "onDemand", "Ease autoscale by schedule or scale factor"
|
15
19
|
class_option :factor, :type => :numeric, :banner => 'scale factor can be decimal too 0.5 for instance'
|
@@ -70,6 +74,8 @@ private
|
|
70
74
|
self.original_config_file = "#{self.backup_folder}/#{self.config_file_name}-#{options[:timestamp]}"
|
71
75
|
|
72
76
|
FileUtils.mkdir_p self.backup_folder
|
77
|
+
|
78
|
+
self.skip_cloudwatch = false
|
73
79
|
end
|
74
80
|
|
75
81
|
def aws_init
|
@@ -77,16 +83,20 @@ private
|
|
77
83
|
ENV['AWS_REGION'] = 'us-east-1'
|
78
84
|
say "using local DynamoDB"
|
79
85
|
self.ddb = Aws::DynamoDB::Client.new({endpoint: 'http://localhost:4567', api: '2012-08-10'})
|
86
|
+
self.skip_cloudwatch = true
|
80
87
|
else
|
81
88
|
credentials = Aws::Credentials.new(
|
82
89
|
self.config['global']['aws-access-key-id'], self.config['global']['aws-secret-access-key-id']
|
83
90
|
)
|
84
91
|
|
85
|
-
|
86
|
-
api: '2012-08-10',
|
92
|
+
aws_config = {
|
87
93
|
region: self.config['global']['region'],
|
88
94
|
credentials: credentials
|
89
|
-
}
|
95
|
+
}
|
96
|
+
|
97
|
+
self.ddb = Aws::DynamoDB::Client.new(aws_config.merge({api: '2012-08-10'}))
|
98
|
+
|
99
|
+
self.cw = Aws::CloudWatch::Client.new(aws_config)
|
90
100
|
end
|
91
101
|
end
|
92
102
|
|
@@ -234,7 +244,7 @@ private
|
|
234
244
|
|
235
245
|
if confirmed
|
236
246
|
|
237
|
-
provisioning ={}
|
247
|
+
provisioning = {}
|
238
248
|
|
239
249
|
active_throughputs = self.config.keys.select{|k| k =~ /table/}
|
240
250
|
active_throughputs.inject(provisioning) { |acc, config_section|
|
@@ -251,14 +261,19 @@ private
|
|
251
261
|
acc[table] ||= {}
|
252
262
|
acc[table]['reads'] = self.config[config_section]['min-provisioned-reads'].to_i
|
253
263
|
acc[table]['writes'] = self.config[config_section]['min-provisioned-writes'].to_i
|
264
|
+
acc[table]['sns-topic-arn'] = self.config[config_section]['sns-topic-arn'] if !self.config[config_section]['sns-topic-arn'].nil?
|
265
|
+
acc[table]['reads-upper-alarm-threshold'] = self.config[config_section]['reads-upper-alarm-threshold'] if !self.config[config_section]['reads-upper-alarm-threshold'].nil?
|
266
|
+
acc[table]['writes-upper-alarm-threshold'] = self.config[config_section]['writes-upper-alarm-threshold'] if !self.config[config_section]['writes-upper-alarm-threshold'].nil?
|
254
267
|
end
|
268
|
+
|
255
269
|
acc
|
256
270
|
}
|
257
271
|
|
258
272
|
log_changes("Update AWS via api call with the following data:\n #{provisioning}\n")
|
259
273
|
|
260
274
|
say "\nWill update: #{provisioning.keys.size} tables\n\n\n", color = :blue
|
261
|
-
|
275
|
+
puts "provisioning: #{provisioning}"
|
276
|
+
|
262
277
|
update_tables(provisioning)
|
263
278
|
else
|
264
279
|
|
@@ -310,19 +325,25 @@ private
|
|
310
325
|
indexes_status
|
311
326
|
end
|
312
327
|
|
313
|
-
def
|
328
|
+
def update!(table_options)
|
329
|
+
puts "table_options: #{table_options}"
|
330
|
+
say "Updating provisioning for table: #{table_options[:table_name]}...", color = :cyan
|
331
|
+
self.ddb.update_table(table_options.reject {|k| k =~ /sns|alarm/})
|
332
|
+
end
|
333
|
+
|
334
|
+
def update_table_if_ready(table_options)
|
314
335
|
while true
|
315
336
|
ready = check_status(table_options[:table_name])
|
316
337
|
|
317
338
|
if ready
|
318
|
-
say "Updating provisioning for table: #{table_options[:table_name]}...", color = :cyan
|
319
339
|
begin
|
320
|
-
result =
|
340
|
+
result = update!(table_options)
|
321
341
|
rescue Exception => e
|
322
342
|
say "\nUnable to update table: #{e.message}\n", color = :red
|
323
343
|
|
324
344
|
if e.message.include?('The requested throughput value equals the current value') || e.message.include?('The requested value equals the current value')
|
325
345
|
say "Skipping table update - the requested throughput value equals the current value", color = :yellow
|
346
|
+
set_cloudwatch_alarms(table_options) unless self.skip_cloudwatch
|
326
347
|
return
|
327
348
|
end
|
328
349
|
|
@@ -333,10 +354,11 @@ private
|
|
333
354
|
end
|
334
355
|
|
335
356
|
say "\nRetrying update on #{table_options[:table_name]}", color = :yellow
|
336
|
-
|
357
|
+
update!(table_options) if e.message.include?('The requested throughput value equals the current value')
|
337
358
|
end
|
338
|
-
return
|
339
359
|
|
360
|
+
set_cloudwatch_alarms(table_options) unless self.skip_cloudwatch
|
361
|
+
return
|
340
362
|
else
|
341
363
|
say "Table #{table_options[:table_name]} is not ready for update, waiting 5 sec before retry", color = :yellow
|
342
364
|
sleep 5
|
@@ -346,32 +368,31 @@ private
|
|
346
368
|
|
347
369
|
def update_tables(provisioning)
|
348
370
|
provisioning.each do |table, values|
|
371
|
+
say "table values before provisioning: #{values}", color = :cyan
|
372
|
+
|
349
373
|
table_options = {
|
350
|
-
|
351
|
-
|
352
|
-
:read_capacity_units => values['reads'],
|
353
|
-
:write_capacity_units => values['writes']
|
354
|
-
}
|
374
|
+
:table_name => table,
|
375
|
+
:provisioned_throughput => {:read_capacity_units => values['reads'], :write_capacity_units => values['writes']}
|
355
376
|
}
|
356
377
|
|
378
|
+
table_options.merge!({:sns_topic_arn => values['sns-topic-arn']}) if values['sns-topic-arn']
|
379
|
+
table_options.merge!({:reads_upper_alarm_threshold => values['reads-upper-alarm-threshold']}) if values['reads-upper-alarm-threshold']
|
380
|
+
table_options.merge!({:writes_upper_alarm_threshold => values['writes-upper-alarm-threshold']}) if values['writes-upper-alarm-threshold']
|
381
|
+
|
357
382
|
# if one of the keys for the table contain index merge the options for update table
|
358
383
|
indexes = provisioning[table].keys.select { |key| key.match(/index/) }
|
359
384
|
if !indexes.empty?
|
360
385
|
indexes.each do |index|
|
361
386
|
table_options.merge!({
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
366
|
-
:write_capacity_units => values[index]['writes']
|
367
|
-
}
|
368
|
-
}
|
369
|
-
}]
|
387
|
+
:global_secondary_index_updates => [{:update => {
|
388
|
+
:index_name => index,
|
389
|
+
:provisioned_throughput => {:read_capacity_units => values[index]['reads'], :write_capacity_units => values[index]['writes']}
|
390
|
+
}}]
|
370
391
|
})
|
371
392
|
end
|
372
393
|
end
|
373
394
|
|
374
|
-
|
395
|
+
update_table_if_ready(table_options)
|
375
396
|
end
|
376
397
|
end
|
377
398
|
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
module ElasticDynamoDb::Cli::CloudWatch
|
4
|
+
|
5
|
+
def set_default_options(hash, member)
|
6
|
+
if hash[member] && !hash[member].nil?
|
7
|
+
default = hash[member]
|
8
|
+
else
|
9
|
+
default = self.config["default_options"][member.to_s.gsub("_", "-")] if !self.config["default_options"][member.to_s.gsub("_", "-")].nil?
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
def put_metric_alarm(table_options, mode, threshold, arn)
|
14
|
+
cw_options = {
|
15
|
+
alarm_name: "#{table_options[:table_name]}-#{mode.capitalize}CapacityUnitsLimit-BasicAlarm",
|
16
|
+
alarm_description: "Consumed#{mode.capitalize}Capacity",
|
17
|
+
actions_enabled: true,
|
18
|
+
ok_actions: [arn],
|
19
|
+
alarm_actions: [arn],
|
20
|
+
metric_name: "Consumed#{mode.capitalize}CapacityUnits",
|
21
|
+
namespace: "AWS/DynamoDB",
|
22
|
+
statistic: "Sum",
|
23
|
+
dimensions: [
|
24
|
+
{
|
25
|
+
name: "TableName",
|
26
|
+
value: table_options[:table_name]
|
27
|
+
}
|
28
|
+
],
|
29
|
+
period: 60,
|
30
|
+
unit: "Count",
|
31
|
+
threshold: threshold,
|
32
|
+
evaluation_periods: 60,
|
33
|
+
comparison_operator: "GreaterThanOrEqualToThreshold"
|
34
|
+
}
|
35
|
+
self.cw.put_metric_alarm(cw_options)
|
36
|
+
end
|
37
|
+
|
38
|
+
def set_cloudwatch_alarms(table_options)
|
39
|
+
arn = set_default_options(table_options, :sns_topic_arn)
|
40
|
+
reads_upper_alarm_threshold = set_default_options(table_options, :reads_upper_alarm_threshold)
|
41
|
+
writes_upper_alarm_threshold = set_default_options(table_options, :writes_upper_alarm_threshold)
|
42
|
+
|
43
|
+
say "Setting CloudWatch alarms thresholds for #{table_options[:table_name]} with upper read threshold of #{reads_upper_alarm_threshold}% and upper writes of #{writes_upper_alarm_threshold}%", color = :magenta
|
44
|
+
|
45
|
+
if !arn.nil? && !reads_upper_alarm_threshold.nil? && !writes_upper_alarm_threshold.nil?
|
46
|
+
put_metric_alarm(table_options, 'read', (table_options[:provisioned_throughput][:read_capacity_units] * 300) * reads_upper_alarm_threshold.to_i/100, arn)
|
47
|
+
put_metric_alarm(table_options, 'writes', (table_options[:provisioned_throughput][:write_capacity_units] * 300) * writes_upper_alarm_threshold.to_i/100, arn)
|
48
|
+
else
|
49
|
+
say "unable to find sns topic in config file - skipping cloudwatch alerts", color = :yellow
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticDynamoDb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: '1.5'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ami Mahloof
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-05-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: configparser
|
@@ -81,6 +81,7 @@ files:
|
|
81
81
|
- elasticDynamoDb.gemspec
|
82
82
|
- lib/elasticDynamoDb.rb
|
83
83
|
- lib/elasticDynamoDb/cli.rb
|
84
|
+
- lib/elasticDynamoDb/cloudwatch.rb
|
84
85
|
- lib/elasticDynamoDb/configparser.rb
|
85
86
|
homepage: https://github.com/innovia/ElasticDynamoDb
|
86
87
|
licenses:
|