elasticDynamoDb 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +15 -0
  2. data/LICENSE +21 -0
  3. data/README.md +50 -0
  4. data/bin/elasticDynamoDb +301 -0
  5. data/lib/configparser.rb +34 -0
  6. metadata +104 -0
checksums.yaml ADDED
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ NjkzOTQ4MjJlODI4MzE0NTFmYTU4MWZjZDgxMjExZjkwZmY3ZjZlYg==
5
+ data.tar.gz: !binary |-
6
+ MjkzYjRiZGYyNjVmNTI1YzMxMTMyNTllYjUwNzAwMWUzNmY0NGM5Yw==
7
+ SHA512:
8
+ metadata.gz: !binary |-
9
+ MGI5ZDM1Mjg1ZjIzMDJlODgzNDBlYWIxN2FkNWY3ZGU4ZTBiNjRkZWNjYjU2
10
+ NmQ0OTY2OGYzMjAxNzM2ZWVlOTllMGQ0NDAyOGI5YjVkYjIwY2U2YTk2MTNh
11
+ NDUyZmZkMDU2MGUyOWI5MWE0ODI4Y2NmOTBkY2JhYzljNWRkZTY=
12
+ data.tar.gz: !binary |-
13
+ ZmYzMTg3NzA3NTU2YjQwYTMyMGYxNzcxYTFjMWU5YzRkNmM5ZDgwZGMxYzQ4
14
+ MGFmYzYxODZmOWFiOGE1OGFjMGZkYjhiYzNhMzQ4OGJkMzdkNGZhMTM4MzJk
15
+ YzBiNTdiNzdmNTQ2ZTdiNGFhMWVjODAyOWEyZWE5MWJhN2I3Mzc=
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2014 Glide Talk, LTD.
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,50 @@
1
+ ## Elastic DynamoDb
2
+
3
+ an OnDemand tool to help with auto scaling of [dynamic-dynamodb](https://github.com/sebdah/dynamic-dynamodb)
4
+
5
+ [dynamic-dynamodb](https://github.com/sebdah/dynamic-dynamodb) tool is great for autoscaling but it has a few limitation:
6
+
7
+ * it does not scale down at once to certian value
8
+
9
+ * it does not accomodate for anticipated traffic spike that can last X hours
10
+
11
+ * it does not scale down to a value at once
12
+
13
+
14
+ ElasticDynamoDb is intended to extend the functionality of dynamic-dynamodb, allowing you to scale by a factor (up/down) and elastically return to the original values it had before
15
+
16
+ it's possible to automate the start and stop of the dynamic-dyanmodb service by passing a bash command (wrapped in quotes) to the --start_cmd / --stop_cmd options
17
+
18
+ ## Installation
19
+ $ gem install elasticDynamoDb
20
+
21
+ Usage:
22
+ ````bash
23
+ elasticDynamoDb onDemand \
24
+ --config-file /etc/dynamic-dynamodb.conf \
25
+ --factor 2 \
26
+ --schedule-restore 120
27
+ --start_cmd "sudo start dynamic-dynamodb" \
28
+ --stop_cmd "sudo stop dynamic-dynamodb"
29
+ ````
30
+
31
+ ````text
32
+ Options:
33
+ --config-file = Location of dynamic-dynamodb.conf
34
+ --factor = Scale factor can be decimal too 0.5 for instance
35
+ [--schedule-restore = Number of minutes for ElasticDynamoDb to restore original values] # Default: 0 (No restore)
36
+ [--working-dir = Location for backup config and change log [default current dir]]
37
+ [--stop-cmd = Bash stop command for dynamic-dynamodb service] (must be wrapped in quotes)
38
+ [--start-cmd = Bash start command for dynamic-dynamodb service] (must be wrapped in quotes)
39
+ ````
40
+
41
+ ## Contributing
42
+
43
+ 1. Fork it
44
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
45
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
46
+ 4. Push to the branch (`git push origin my-new-feature`)
47
+ 5. Create new Pull Request
48
+
49
+ ## License
50
+ ElasticDynamoDB is released under [MIT License](http://www.opensource.org/licenses/MIT)
@@ -0,0 +1,301 @@
1
+ #!/usr/bin/env ruby
2
+ require 'thor'
3
+ require 'aws-sdk'
4
+ require 'fileutils'
5
+ require_relative '../lib/configparser'
6
+
7
+ class ElasticDynamoDb < Thor
8
+ default_task :show_help
9
+ include Thor::Actions
10
+
11
+ desc "onDemand", "Ease autoscale by schedule or scale factor"
12
+ method_option :factor, :required => true, :alias => 'f', :type => :numeric, :banner => 'scale factor can be decimal too 0.5 for instance'
13
+ method_option :schedule_restore, :alias => 's', :type => :numeric, :banner => 'number of minutes for ElasticDynamoDb to restore original values', :default => 0
14
+ method_option :working_dir, :default => '.', :alais => 'd', :banner => 'location for backup config and change log [default current dir]'
15
+ method_option :config_file, :required => true, :alias => 'c', :banner => 'location of dynamic-dynamodb.conf'
16
+ method_option :stop_cmd, :banner => 'bash stop command for dynamic-dynamodb service'
17
+ method_option :start_cmd, :banner => 'bash start command for dynamic-dynamodb service'
18
+ def onDemand
19
+ if dynamic_dynamodb_running?
20
+ say("Dynamic-DynamoDB is running - no point in changing values while old config is in memory, stop the service and re-run ElasticDynamoDb", color=:red)
21
+ exit
22
+ end
23
+
24
+ @@config_file = options[:config_file]
25
+ @@in_restore = false
26
+ @@timestamp = Time.now.strftime("%m%d%Y-%H%M%S")
27
+
28
+ @@working_dir = options[:working_dir]
29
+ @@bkp_folder = "#{File.expand_path(@@working_dir)}/ElasticDynamoDb/dynamodb_config_backups"
30
+ @@log_file = "#{File.expand_path(@@working_dir)}/ElasticDynamoDb/change.log"
31
+ @@original_config_file = "#{@@bkp_folder}/#{@@config_file}-#{@@timestamp}"
32
+
33
+ FileUtils.mkdir_p "#{File.expand_path(@@working_dir)}/ElasticDynamoDb/dynamodb_config_backups"
34
+
35
+ process_config(@@config_file, options[:factor])
36
+
37
+ if options[:schedule_restore] > 0 && !@@in_restore
38
+ say("#{Time.now} - Waiting here for #{options[:schedule_restore]} minutes until restore")
39
+ sleep options[:schedule_restore] * 60
40
+ @@in_restore = true
41
+ say "#{Time.now} - Restoring to original config file (#{@@original_config_file}))"
42
+ process_config(@@original_config_file, 1)
43
+ end
44
+
45
+ say("All done! you may restart the dynamic-dynamodb process")
46
+ end
47
+
48
+ desc 'show_help', 'Display Usage'
49
+ def show_help
50
+ help_block = <<-HELP
51
+ Elastic DynamoDb - an OnDemand tool to help with auto scaling of dynamic-dynamodb
52
+ Synopsis:
53
+ dynamic-dynamodb tool is great for autoscaling, however it does not scale down at once to certian value
54
+ and it does not accomodate for anticipated traffic spike that can last X hours
55
+
56
+ This tool is intended to extend the functionality of dynamic-dynamodb, allowing you to scale by a factor (up/down) and elastically return to the original values it had before
57
+
58
+ it possible to automate the start and stop of the service by passing a bash command to the --start_cmd / --stop_cmd
59
+
60
+ Usage:
61
+ elasticDynamoDb onDemand --config-file=location of dynamic-dynamodb.conf --factor=Scale factor (can be decimal too, i.e: 0.5) --schedule-restore 120
62
+
63
+ Options:
64
+ --config-file=location of dynamic-dynamodb.conf
65
+ --factor scale factor can be decimal too 0.5 for instance
66
+ [--schedule-restore=number of minutes for ElasticDynamoDb to restore original values] # Default: 0 (No restore)
67
+ [--working-dir=location for backup config and change log [default current dir]]
68
+ [--stop-cmd=bash stop command for dynamic-dynamodb service (must be wrapped in quotes)]
69
+ [--start-cmd=bash start command for dynamic-dynamodb service (must be wrapped in quotes)]
70
+ HELP
71
+ say(help_block)
72
+ end
73
+
74
+ private
75
+ def process_config(file, factor)
76
+ read_config(file)
77
+ scale(factor)
78
+ write_config(factor)
79
+ system(stop_cmd) if options[:stop_cmd]
80
+ update_aws_api
81
+ system(start_cmd) if options[:start_cmd]
82
+ end
83
+
84
+ def dynamic_dynamodb_running?
85
+ dynamic_dynamodb_pid = `ps aux | grep -v grep | grep -i dynamic-dynamodb | awk {'print $2'}`.to_i
86
+ return Dir['/proc/[0-9]*'].map { |i| i.match(/\d+/)[0].to_i }.include?(dynamic_dynamodb_pid)
87
+ end
88
+
89
+ def read_config(config_file)
90
+ @@config = ConfigParser.new(config_file).parse
91
+ end
92
+
93
+ def scale(factor=nil)
94
+ if !factor.nil?
95
+ scale_factor = factor
96
+
97
+ active_throughputs = @@config.keys.select{|k| k =~ /table/}
98
+
99
+ active_throughputs.each do |prefix|
100
+ min_reads = @@config[prefix]['min-provisioned-reads'].to_i
101
+ min_writes = @@config[prefix]['min-provisioned-writes'].to_i
102
+
103
+ say("(scale factor: #{scale_factor}) Global Secondary Index / Table: #{prefix.gsub('$','').gsub('^', '')} =>", color=:cyan)
104
+ puts "Current min-read from #{@@config_file}: #{min_reads}"
105
+ puts "Current min-write from #{@@config_file}: #{min_writes}"
106
+
107
+ @@config[prefix]['min-provisioned-reads'] = (min_reads * scale_factor).to_i
108
+ @@config[prefix]['min-provisioned-writes'] = (min_writes * scale_factor).to_i
109
+
110
+ say("New min reads: #{@@config[prefix]['min-provisioned-reads']}", color=:yellow)
111
+ say("New min writes: #{@@config[prefix]['min-provisioned-writes']}", color=:yellow)
112
+ puts "------------------------------------------------"
113
+ end
114
+ else
115
+ say("Need a factor to scale by,(i.e scale --factor 2)", color=:green)
116
+ exit
117
+ end
118
+ end
119
+
120
+ def write_config(scale_factor)
121
+ if options[:schedule_restore] > 0
122
+ restore = "auto restore to bakcup config file (#{@@original_config_file}) in #{options[:schedule_restore]} minutes"
123
+ else
124
+ restore = "Backup to #{@@original_config_file}"
125
+ end
126
+
127
+ if @@in_restore
128
+ confirmed = true
129
+ else
130
+ confirmed = yes?("OverWrite the new config file (#{restore})? (yes/no)", color=:red)
131
+ end
132
+
133
+ if confirmed
134
+ backup
135
+
136
+ str_to_write = ''
137
+ @@config.each do |section, lines|
138
+ str_to_write += "[#{section}]\n"
139
+ lines.each do |line, value|
140
+ if line =~ /^(#|;|\n)/
141
+ str_to_write += "#{line}\n"
142
+ else
143
+ str_to_write += "#{line}: #{value}\n"
144
+ end
145
+ end
146
+ end
147
+
148
+ save_file(str_to_write)
149
+
150
+ if !@@in_restore
151
+ reason = ask('type the reason for the change: ', color=:magenta)
152
+ else
153
+ reason = 'Auto restore to @@original_config_file'
154
+ end
155
+
156
+ log_changes("#{reason} - Changed throughputs by factor of: #{scale_factor}")
157
+
158
+ say("new config changes commited to file")
159
+ else
160
+ say("Not doing antything - Goodbye...", color=:green)
161
+ exit
162
+ end
163
+ end
164
+
165
+ def log_changes(msg)
166
+ say("Recording change in #{@@log_file}")
167
+
168
+ File.open(@@log_file, 'a') do |file|
169
+ file.write "#{Time.now} - #{msg}\n"
170
+ end
171
+ end
172
+
173
+ def backup
174
+ `mkdir -p #{@@bkp_folder} && cp #{@@config_file} #{@@original_config_file}` if @@config_file
175
+ say "backup file created: #{@@original_config_file}"
176
+ end
177
+
178
+ def save_file(str)
179
+ File.open(@@config_file, 'w') do |file|
180
+ file.write str
181
+ end
182
+ end
183
+
184
+ def update_aws_api
185
+ if @@in_restore
186
+ confirmed = true
187
+ else
188
+ confirmed = yes?('Update all tables with these values on DynamoDb? (yes/no)', color=:red)
189
+ end
190
+
191
+ if confirmed
192
+ AWS.config({
193
+ :access_key_id => @@config['global']['aws-access-key-id'],
194
+ :secret_access_key => @@config['global']['aws-secret-access-key-id'],
195
+ :region => @@config['global']['region'],
196
+ })
197
+
198
+ @@ddb = AWS::DynamoDB::Client.new(:api_version => '2012-08-10')
199
+
200
+ provisioning ={}
201
+
202
+ active_throughputs = @@config.keys.select{|k| k =~ /table/}
203
+
204
+ active_throughputs.inject(provisioning) { |acc, config_section|
205
+ if config_section.include?('index')
206
+ config_section =~ /^gsi: \^(.*)\$\stable: \^(.*)\$$/
207
+ index = $1
208
+ table = $2
209
+
210
+ acc[table] ||= {}
211
+ acc[table][index] ||= {}
212
+ acc[table][index]['reads'] = @@config[config_section]['min-provisioned-reads'].to_i
213
+ acc[table][index]['writes'] = @@config[config_section]['min-provisioned-writes'].to_i
214
+ else
215
+ config_section =~ /^table:?\S\^(.*)\$/
216
+ table = $1
217
+ acc[table] ||= {}
218
+ acc[table]['reads'] = @@config[config_section]['min-provisioned-reads'].to_i
219
+ acc[table]['writes'] = @@config[config_section]['min-provisioned-writes'].to_i
220
+ end
221
+ acc
222
+ }
223
+ update_tables(provisioning)
224
+ log_changes("Update AWS via api call with the following data:\n #{provisioning}\n")
225
+ end
226
+ end
227
+
228
+ def check_status(table_name)
229
+ until table_status(table_name) == 'ACTIVE' && !indexes_status(table_name).any? {|i| i != 'ACTIVE'}
230
+ say("#{table_name} is not ACTIVE => sleeping for 30 sec and will retry again", color=:yellow)
231
+ sleep 30
232
+ end
233
+ return true
234
+ end
235
+
236
+ def table_status(table_name)
237
+ print "Checking table #{table_name} status..."
238
+ status = @@ddb.describe_table({:table_name => table_name}).table.table_status
239
+ puts status
240
+ status
241
+ end
242
+
243
+ def indexes_status(table_name)
244
+ print "Checking indexes status on #{table_name}..."
245
+ indexes_status = []
246
+ if @@ddb.describe_table({:table_name => table_name}).table.keys.include?(:global_secondary_indexes)
247
+ @@ddb.describe_table({:table_name => table_name}).table.global_secondary_indexes.each {|i| indexes_status << i.index_status }
248
+ end
249
+
250
+ if indexes_status.empty?
251
+ puts "No indexes for #{table_name} table"
252
+ else
253
+ puts indexes_status
254
+ end
255
+ indexes_status
256
+ end
257
+
258
+ def update_tables(provisioning)
259
+ provisioning.each do |table, values|
260
+ options = {
261
+ :table_name => table,
262
+ :provisioned_throughput => {
263
+ :read_capacity_units => values['reads'],
264
+ :write_capacity_units => values['writes']
265
+ }
266
+ }
267
+
268
+ # if one of the keys for the table contain index merge the options for update table
269
+ indexes = provisioning[table].keys.select { |key| key.match(/index/) }
270
+ if !indexes.empty?
271
+ indexes.each do |index|
272
+ options.merge!({
273
+ :global_secondary_index_updates => [{:update => {
274
+ :index_name => index,
275
+ :provisioned_throughput => {
276
+ :read_capacity_units => values[index]['reads'],
277
+ :write_capacity_units => values[index]['writes']
278
+ }
279
+ }
280
+ }]
281
+ })
282
+ end
283
+ end
284
+
285
+ ready = check_status(table)
286
+
287
+ begin
288
+ say("Updating provisioning for table: #{table}",color=:cyan)
289
+ @@ddb.update_table(options) if ready
290
+ rescue Exception => e
291
+ if e.to_s.include?('must be at most 100 percent of current value')
292
+ say("No Change - Scale up is higher then 100% of the original values, restrating dynamic-dynamodb process will auto scale up to the new values", color=:yellow)
293
+ else
294
+ say("unable to update table: #{e}", color=:red)
295
+ end
296
+ end
297
+ end
298
+ end
299
+ end
300
+
301
+ ElasticDynamoDb.start
@@ -0,0 +1,34 @@
1
+ class ConfigParser < Hash
2
+ def initialize(fname = nil)
3
+ @file_name = fname
4
+ end
5
+
6
+ def parse
7
+ begin
8
+ input_source = File.open(@file_name, "r").each_line if @file_name
9
+ rescue Exception => e
10
+ puts "error loading file -> #{e}"
11
+ end
12
+ config = {}
13
+ section = nil
14
+ key = nil
15
+
16
+ input_source.inject(config) {|acc, line|
17
+ acc ||= {}
18
+
19
+ if line =~ /^\[(.+?)\]/ # if line contain section start adding it to a new hash key
20
+ section = $1
21
+ acc[section] ||= {}
22
+ elsif line =~ /^(#|;|\n)/
23
+ acc[section]["#{line}"] = line
24
+ elsif line =~ /^\s*(.+?)\s*[=:]\s*(.+)$/ # this returns a 2 parts $1 first response, $2 2nd response
25
+ key = $1
26
+ value = $2
27
+ acc[section] ||= {}
28
+ acc[section][key] = value
29
+ end
30
+
31
+ acc
32
+ }
33
+ end
34
+ end
metadata ADDED
@@ -0,0 +1,104 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: elasticDynamoDb
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Ami Mahloof
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-08-07 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: configparser
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: thor
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: '0.19'
34
+ - - ! '>='
35
+ - !ruby/object:Gem::Version
36
+ version: 0.19.1
37
+ type: :runtime
38
+ prerelease: false
39
+ version_requirements: !ruby/object:Gem::Requirement
40
+ requirements:
41
+ - - ~>
42
+ - !ruby/object:Gem::Version
43
+ version: '0.19'
44
+ - - ! '>='
45
+ - !ruby/object:Gem::Version
46
+ version: 0.19.1
47
+ - !ruby/object:Gem::Dependency
48
+ name: aws-sdk
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - ~>
52
+ - !ruby/object:Gem::Version
53
+ version: '1.40'
54
+ - - ! '>='
55
+ - !ruby/object:Gem::Version
56
+ version: 1.40.0
57
+ type: :runtime
58
+ prerelease: false
59
+ version_requirements: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - ~>
62
+ - !ruby/object:Gem::Version
63
+ version: '1.40'
64
+ - - ! '>='
65
+ - !ruby/object:Gem::Version
66
+ version: 1.40.0
67
+ description: scale dynamodb by factor with dynamic-dynamodb
68
+ email: ami.mahloof@gmail.com
69
+ executables:
70
+ - elasticDynamoDb
71
+ extensions: []
72
+ extra_rdoc_files:
73
+ - README.md
74
+ - LICENSE
75
+ files:
76
+ - LICENSE
77
+ - README.md
78
+ - bin/elasticDynamoDb
79
+ - lib/configparser.rb
80
+ homepage: https://github.com/innovia
81
+ licenses:
82
+ - MIT
83
+ metadata: {}
84
+ post_install_message:
85
+ rdoc_options: []
86
+ require_paths:
87
+ - lib
88
+ required_ruby_version: !ruby/object:Gem::Requirement
89
+ requirements:
90
+ - - ! '>='
91
+ - !ruby/object:Gem::Version
92
+ version: '0'
93
+ required_rubygems_version: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - ! '>='
96
+ - !ruby/object:Gem::Version
97
+ version: 1.3.6
98
+ requirements: []
99
+ rubyforge_project:
100
+ rubygems_version: 2.2.2
101
+ signing_key:
102
+ specification_version: 4
103
+ summary: scale dynamodb by factor with dynamic-dynamodb
104
+ test_files: []