dynamo-record 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.dockerignore +15 -0
- data/.gitignore +9 -5
- data/.rspec +1 -0
- data/.rubocop.yml +9 -30
- data/.travis.yml +18 -2
- data/Dockerfile +22 -0
- data/Gemfile +0 -1
- data/LICENSE.txt +21 -0
- data/README.md +75 -17
- data/build.sh +10 -20
- data/docker-compose.override.example.yml +19 -0
- data/docker-compose.yml +7 -2
- data/dynamo-record.gemspec +40 -28
- data/lib/dynamo/record.rb +17 -0
- data/lib/dynamo/record/marshalers.rb +46 -0
- data/lib/dynamo/record/model.rb +127 -0
- data/lib/dynamo/record/model_existence_validator.rb +10 -0
- data/lib/{dynamo-record → dynamo}/record/railtie.rb +1 -3
- data/lib/dynamo/record/table_migration.rb +59 -0
- data/lib/dynamo/record/task_helpers/cleanup.rb +21 -0
- data/lib/dynamo/record/task_helpers/drop_all_tables.rb +19 -0
- data/lib/dynamo/record/task_helpers/drop_table.rb +13 -0
- data/lib/dynamo/record/task_helpers/list_tables.rb +15 -0
- data/lib/dynamo/record/task_helpers/migration_runner.rb +72 -0
- data/lib/dynamo/record/task_helpers/scale.rb +90 -0
- data/lib/dynamo/record/version.rb +5 -0
- data/lib/tasks/dynamo.rake +7 -7
- metadata +96 -37
- data/Dockerfile.test +0 -23
- data/Gemfile.lock +0 -178
- data/doc/testing.md +0 -11
- data/docker-compose.dev.override.yml +0 -6
- data/lib/dynamo-record.rb +0 -4
- data/lib/dynamo-record/marshalers.rb +0 -44
- data/lib/dynamo-record/model.rb +0 -127
- data/lib/dynamo-record/record.rb +0 -7
- data/lib/dynamo-record/record/version.rb +0 -5
- data/lib/dynamo-record/table_migration.rb +0 -58
- data/lib/dynamo-record/task_helpers/cleanup.rb +0 -19
- data/lib/dynamo-record/task_helpers/drop_all_tables.rb +0 -17
- data/lib/dynamo-record/task_helpers/drop_table.rb +0 -11
- data/lib/dynamo-record/task_helpers/list_tables.rb +0 -13
- data/lib/dynamo-record/task_helpers/migration_runner.rb +0 -70
- data/lib/dynamo-record/task_helpers/scale.rb +0 -86
- data/lib/model_existence_validator.rb +0 -7
@@ -0,0 +1,72 @@
|
|
1
|
+
module Dynamo
|
2
|
+
module Record
|
3
|
+
module TaskHelpers
|
4
|
+
class MigrationRunner
|
5
|
+
def self.run(path = 'db/dynamo_migrate')
|
6
|
+
constants = []
|
7
|
+
filename_regexp = /\A([0-9]+)_([_a-z0-9]*)\.?([_a-z0-9]*)?\.rb\z/
|
8
|
+
|
9
|
+
# Sorts the files located in `db/dynamo_migrate` to ensure order is preserved
|
10
|
+
Dir[Rails.root.join("#{path}/*.rb")].sort.each do |f|
|
11
|
+
migration = migration(f, filename_regexp, constants)
|
12
|
+
|
13
|
+
# starts the migration
|
14
|
+
yield "Migrating: #{migration}"
|
15
|
+
|
16
|
+
begin
|
17
|
+
status = table_config_check(migration)
|
18
|
+
yield status if status
|
19
|
+
|
20
|
+
status = up(migration)
|
21
|
+
yield status if status
|
22
|
+
|
23
|
+
status = update(migration)
|
24
|
+
yield status if status
|
25
|
+
rescue StandardError => e
|
26
|
+
yield "Migration failed: #{e}"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def self.migration(f, filename_regexp, constants)
|
32
|
+
raise "Non-numeric prefix: #{f}" if File.basename(f).scan(filename_regexp).first.nil?
|
33
|
+
require f
|
34
|
+
|
35
|
+
# finds the constant that was added on the require statement above
|
36
|
+
migration_sym = (DynamoMigrate.constants - constants).first
|
37
|
+
migration = DynamoMigrate.const_get(migration_sym)
|
38
|
+
constants.push migration_sym
|
39
|
+
migration
|
40
|
+
end
|
41
|
+
|
42
|
+
def self.status_message(status)
|
43
|
+
case status
|
44
|
+
when :exists
|
45
|
+
'Table already exists'
|
46
|
+
when :migrated
|
47
|
+
'Migration successful'
|
48
|
+
else
|
49
|
+
raise 'Migration failed'
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def self.up(migration)
|
54
|
+
return unless migration.respond_to? :up
|
55
|
+
status_message migration.up
|
56
|
+
end
|
57
|
+
|
58
|
+
def self.table_config_check(migration)
|
59
|
+
return unless migration.respond_to? :table_config
|
60
|
+
status_message migration.table_config_check
|
61
|
+
end
|
62
|
+
|
63
|
+
def self.update(migration)
|
64
|
+
return unless migration.respond_to? :update
|
65
|
+
status = migration.update
|
66
|
+
return 'Migration successful' if status == :updated
|
67
|
+
status
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
module Dynamo
|
2
|
+
module Record
|
3
|
+
module TaskHelpers
|
4
|
+
class Scale
|
5
|
+
attr_reader :model, :attribute_selector, :new_throughput
|
6
|
+
attr_reader :migration, :existing_throughput, :model_name
|
7
|
+
|
8
|
+
def initialize(model_name, attribute_selector, new_throughput)
|
9
|
+
@model_name = model_name
|
10
|
+
@attribute_selector = attribute_selector
|
11
|
+
@new_throughput = new_throughput
|
12
|
+
end
|
13
|
+
|
14
|
+
def run
|
15
|
+
return description if [model_name, attribute_selector, new_throughput].any?(&:nil?)
|
16
|
+
|
17
|
+
@model = model_name.constantize
|
18
|
+
@migration = Aws::Record::TableMigration.new(model)
|
19
|
+
@existing_throughput = model.provisioned_throughput
|
20
|
+
|
21
|
+
update_throughput
|
22
|
+
success_message
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def success_message
|
28
|
+
"Successfully updated #{model.table_name} throughput to #{update_instructions[:provisioned_throughput]}"
|
29
|
+
end
|
30
|
+
|
31
|
+
def update_throughput
|
32
|
+
raise_attribute_error if !read? && !write?
|
33
|
+
|
34
|
+
migration.update!(update_instructions)
|
35
|
+
end
|
36
|
+
|
37
|
+
def update_instructions
|
38
|
+
{
|
39
|
+
provisioned_throughput: {
|
40
|
+
write_capacity_units: (write? && new_throughput) || existing_write,
|
41
|
+
read_capacity_units: (read? && new_throughput) || existing_read
|
42
|
+
}
|
43
|
+
}
|
44
|
+
end
|
45
|
+
|
46
|
+
def existing_write
|
47
|
+
existing_throughput[:write_capacity_units]
|
48
|
+
end
|
49
|
+
|
50
|
+
def existing_read
|
51
|
+
existing_throughput[:read_capacity_units]
|
52
|
+
end
|
53
|
+
|
54
|
+
def both?
|
55
|
+
attribute_selector.to_sym == :both
|
56
|
+
end
|
57
|
+
|
58
|
+
def read?
|
59
|
+
both? || attribute_selector.to_sym == :read
|
60
|
+
end
|
61
|
+
|
62
|
+
def write?
|
63
|
+
both? || attribute_selector.to_sym == :write
|
64
|
+
end
|
65
|
+
|
66
|
+
def raise_attribute_error
|
67
|
+
raise ArgumentError, <<~MESSAGE
|
68
|
+
You didn't provide an appropriate attribute selection. We accept [:both, :read, :write]
|
69
|
+
MESSAGE
|
70
|
+
end
|
71
|
+
|
72
|
+
def description
|
73
|
+
<<~DESCRIPTION
|
74
|
+
----------------------------------------------------------------------
|
75
|
+
Here's some usage information:
|
76
|
+
Scale a dynamo table. Requires three inputs.
|
77
|
+
- ModelName
|
78
|
+
ruby class of the model
|
79
|
+
- attribute
|
80
|
+
valid values include "both", "read" and "write"
|
81
|
+
- new_throughput
|
82
|
+
numerical value for the new read/write capacity units
|
83
|
+
Example: `rake dynamo:scale[MySuperDynamoModel,both,50]`
|
84
|
+
----------------------------------------------------------------------
|
85
|
+
DESCRIPTION
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
data/lib/tasks/dynamo.rake
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
namespace :dynamo do
|
2
2
|
desc 'Run dynamo migrations'
|
3
3
|
task migrate: :environment do
|
4
|
-
|
4
|
+
Dynamo::Record::TaskHelpers::MigrationRunner.run { |msg| puts msg }
|
5
5
|
end
|
6
6
|
|
7
7
|
desc 'Drops all dynamo tables and re-runs migrations'
|
@@ -12,27 +12,27 @@ namespace :dynamo do
|
|
12
12
|
|
13
13
|
desc 'Drop all dynamo tables'
|
14
14
|
task :drop_all, [:override] => :environment do |_t, args|
|
15
|
-
puts
|
15
|
+
puts Dynamo::Record::TaskHelpers::DropAllTables.run args[:override]
|
16
16
|
end
|
17
17
|
|
18
18
|
desc 'Drop a specified dynamo table'
|
19
19
|
task :drop, [:table_name] => :environment do |_t, args|
|
20
|
-
puts
|
20
|
+
puts Dynamo::Record::TaskHelpers::DropTable.run args[:table_name]
|
21
21
|
end
|
22
22
|
|
23
23
|
desc 'List all tables with dynamo prefix'
|
24
24
|
task list_tables: :environment do
|
25
|
-
puts
|
25
|
+
puts Dynamo::Record::TaskHelpers::ListTables.run
|
26
26
|
end
|
27
27
|
|
28
28
|
desc 'Delete all records in all DyanmoDB tables.'
|
29
29
|
task cleanup: :environment do
|
30
|
-
|
30
|
+
Dynamo::Record::TaskHelpers::Cleanup.run
|
31
31
|
puts 'Finished deleting all records in all DynamoDB tables.'
|
32
32
|
end
|
33
33
|
|
34
34
|
desc 'Scale a dynamo table read/write capacity to the new capacity value provided'
|
35
|
-
task :scale, [
|
36
|
-
puts
|
35
|
+
task :scale, %i[model_name attribute new_throughput] => :environment do |_t, args|
|
36
|
+
puts Dynamo::Record::TaskHelpers::Scale.new(args[:model_name], args[:attribute], args[:new_throughput].to_i).run
|
37
37
|
end
|
38
38
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dynamo-record
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Davis McClellan
|
@@ -12,9 +12,9 @@ authors:
|
|
12
12
|
- Augusto Callejas
|
13
13
|
- Frank Murphy
|
14
14
|
autorequire:
|
15
|
-
bindir:
|
15
|
+
bindir: bin
|
16
16
|
cert_chain: []
|
17
|
-
date: 2017-
|
17
|
+
date: 2017-09-22 00:00:00.000000000 Z
|
18
18
|
dependencies:
|
19
19
|
- !ruby/object:Gem::Dependency
|
20
20
|
name: aws-record
|
@@ -31,33 +31,79 @@ dependencies:
|
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '1.1'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
name:
|
34
|
+
name: activemodel
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
|
-
- - "
|
37
|
+
- - ">="
|
38
38
|
- !ruby/object:Gem::Version
|
39
39
|
version: '4.2'
|
40
|
+
- - "<"
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: '5.2'
|
40
43
|
type: :runtime
|
41
44
|
prerelease: false
|
42
45
|
version_requirements: !ruby/object:Gem::Requirement
|
43
46
|
requirements:
|
44
|
-
- - "
|
47
|
+
- - ">="
|
45
48
|
- !ruby/object:Gem::Version
|
46
49
|
version: '4.2'
|
50
|
+
- - "<"
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: '5.2'
|
53
|
+
- !ruby/object:Gem::Dependency
|
54
|
+
name: railties
|
55
|
+
requirement: !ruby/object:Gem::Requirement
|
56
|
+
requirements:
|
57
|
+
- - ">="
|
58
|
+
- !ruby/object:Gem::Version
|
59
|
+
version: '4.2'
|
60
|
+
- - "<"
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '5.2'
|
63
|
+
type: :runtime
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '4.2'
|
70
|
+
- - "<"
|
71
|
+
- !ruby/object:Gem::Version
|
72
|
+
version: '5.2'
|
73
|
+
- !ruby/object:Gem::Dependency
|
74
|
+
name: activesupport
|
75
|
+
requirement: !ruby/object:Gem::Requirement
|
76
|
+
requirements:
|
77
|
+
- - ">="
|
78
|
+
- !ruby/object:Gem::Version
|
79
|
+
version: '4.2'
|
80
|
+
- - "<"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '5.2'
|
83
|
+
type: :development
|
84
|
+
prerelease: false
|
85
|
+
version_requirements: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '4.2'
|
90
|
+
- - "<"
|
91
|
+
- !ruby/object:Gem::Version
|
92
|
+
version: '5.2'
|
47
93
|
- !ruby/object:Gem::Dependency
|
48
94
|
name: bundler
|
49
95
|
requirement: !ruby/object:Gem::Requirement
|
50
96
|
requirements:
|
51
97
|
- - "~>"
|
52
98
|
- !ruby/object:Gem::Version
|
53
|
-
version: '1.
|
99
|
+
version: '1.15'
|
54
100
|
type: :development
|
55
101
|
prerelease: false
|
56
102
|
version_requirements: !ruby/object:Gem::Requirement
|
57
103
|
requirements:
|
58
104
|
- - "~>"
|
59
105
|
- !ruby/object:Gem::Version
|
60
|
-
version: '1.
|
106
|
+
version: '1.15'
|
61
107
|
- !ruby/object:Gem::Dependency
|
62
108
|
name: byebug
|
63
109
|
requirement: !ruby/object:Gem::Requirement
|
@@ -92,56 +138,56 @@ dependencies:
|
|
92
138
|
requirements:
|
93
139
|
- - "~>"
|
94
140
|
- !ruby/object:Gem::Version
|
95
|
-
version: '
|
141
|
+
version: '12.0'
|
96
142
|
type: :development
|
97
143
|
prerelease: false
|
98
144
|
version_requirements: !ruby/object:Gem::Requirement
|
99
145
|
requirements:
|
100
146
|
- - "~>"
|
101
147
|
- !ruby/object:Gem::Version
|
102
|
-
version: '
|
148
|
+
version: '12.0'
|
103
149
|
- !ruby/object:Gem::Dependency
|
104
150
|
name: rspec
|
105
151
|
requirement: !ruby/object:Gem::Requirement
|
106
152
|
requirements:
|
107
153
|
- - "~>"
|
108
154
|
- !ruby/object:Gem::Version
|
109
|
-
version: '3.
|
155
|
+
version: '3.6'
|
110
156
|
type: :development
|
111
157
|
prerelease: false
|
112
158
|
version_requirements: !ruby/object:Gem::Requirement
|
113
159
|
requirements:
|
114
160
|
- - "~>"
|
115
161
|
- !ruby/object:Gem::Version
|
116
|
-
version: '3.
|
162
|
+
version: '3.6'
|
117
163
|
- !ruby/object:Gem::Dependency
|
118
164
|
name: rubocop
|
119
165
|
requirement: !ruby/object:Gem::Requirement
|
120
166
|
requirements:
|
121
167
|
- - "~>"
|
122
168
|
- !ruby/object:Gem::Version
|
123
|
-
version: 0.
|
169
|
+
version: 0.50.0
|
124
170
|
type: :development
|
125
171
|
prerelease: false
|
126
172
|
version_requirements: !ruby/object:Gem::Requirement
|
127
173
|
requirements:
|
128
174
|
- - "~>"
|
129
175
|
- !ruby/object:Gem::Version
|
130
|
-
version: 0.
|
176
|
+
version: 0.50.0
|
131
177
|
- !ruby/object:Gem::Dependency
|
132
178
|
name: simplecov
|
133
179
|
requirement: !ruby/object:Gem::Requirement
|
134
180
|
requirements:
|
135
181
|
- - "~>"
|
136
182
|
- !ruby/object:Gem::Version
|
137
|
-
version: '0.
|
183
|
+
version: '0.14'
|
138
184
|
type: :development
|
139
185
|
prerelease: false
|
140
186
|
version_requirements: !ruby/object:Gem::Requirement
|
141
187
|
requirements:
|
142
188
|
- - "~>"
|
143
189
|
- !ruby/object:Gem::Version
|
144
|
-
version: '0.
|
190
|
+
version: '0.14'
|
145
191
|
- !ruby/object:Gem::Dependency
|
146
192
|
name: webmock
|
147
193
|
requirement: !ruby/object:Gem::Requirement
|
@@ -156,7 +202,21 @@ dependencies:
|
|
156
202
|
- - "~>"
|
157
203
|
- !ruby/object:Gem::Version
|
158
204
|
version: '2.1'
|
159
|
-
|
205
|
+
- !ruby/object:Gem::Dependency
|
206
|
+
name: wwtd
|
207
|
+
requirement: !ruby/object:Gem::Requirement
|
208
|
+
requirements:
|
209
|
+
- - "~>"
|
210
|
+
- !ruby/object:Gem::Version
|
211
|
+
version: '1.3'
|
212
|
+
type: :development
|
213
|
+
prerelease: false
|
214
|
+
version_requirements: !ruby/object:Gem::Requirement
|
215
|
+
requirements:
|
216
|
+
- - "~>"
|
217
|
+
- !ruby/object:Gem::Version
|
218
|
+
version: '1.3'
|
219
|
+
description:
|
160
220
|
email:
|
161
221
|
- dmcclellan@instructure.com
|
162
222
|
- rtaylor@instructure.com
|
@@ -169,35 +229,34 @@ executables: []
|
|
169
229
|
extensions: []
|
170
230
|
extra_rdoc_files: []
|
171
231
|
files:
|
232
|
+
- ".dockerignore"
|
172
233
|
- ".gitignore"
|
173
234
|
- ".rspec"
|
174
235
|
- ".rubocop.yml"
|
175
236
|
- ".travis.yml"
|
176
|
-
- Dockerfile
|
237
|
+
- Dockerfile
|
177
238
|
- Gemfile
|
178
|
-
-
|
239
|
+
- LICENSE.txt
|
179
240
|
- README.md
|
180
241
|
- build.sh
|
181
|
-
-
|
182
|
-
- docker-compose.dev.override.yml
|
242
|
+
- docker-compose.override.example.yml
|
183
243
|
- docker-compose.yml
|
184
244
|
- dynamo-record.gemspec
|
185
|
-
- lib/dynamo
|
186
|
-
- lib/dynamo
|
187
|
-
- lib/dynamo
|
188
|
-
- lib/dynamo
|
189
|
-
- lib/dynamo
|
190
|
-
- lib/dynamo
|
191
|
-
- lib/dynamo
|
192
|
-
- lib/dynamo
|
193
|
-
- lib/dynamo
|
194
|
-
- lib/dynamo
|
195
|
-
- lib/dynamo
|
196
|
-
- lib/dynamo
|
197
|
-
- lib/dynamo
|
198
|
-
- lib/model_existence_validator.rb
|
245
|
+
- lib/dynamo/record.rb
|
246
|
+
- lib/dynamo/record/marshalers.rb
|
247
|
+
- lib/dynamo/record/model.rb
|
248
|
+
- lib/dynamo/record/model_existence_validator.rb
|
249
|
+
- lib/dynamo/record/railtie.rb
|
250
|
+
- lib/dynamo/record/table_migration.rb
|
251
|
+
- lib/dynamo/record/task_helpers/cleanup.rb
|
252
|
+
- lib/dynamo/record/task_helpers/drop_all_tables.rb
|
253
|
+
- lib/dynamo/record/task_helpers/drop_table.rb
|
254
|
+
- lib/dynamo/record/task_helpers/list_tables.rb
|
255
|
+
- lib/dynamo/record/task_helpers/migration_runner.rb
|
256
|
+
- lib/dynamo/record/task_helpers/scale.rb
|
257
|
+
- lib/dynamo/record/version.rb
|
199
258
|
- lib/tasks/dynamo.rake
|
200
|
-
homepage: https://
|
259
|
+
homepage: https://github.com/instructure/dynamo-record
|
201
260
|
licenses:
|
202
261
|
- MIT
|
203
262
|
metadata: {}
|
@@ -220,5 +279,5 @@ rubyforge_project:
|
|
220
279
|
rubygems_version: 2.6.11
|
221
280
|
signing_key:
|
222
281
|
specification_version: 4
|
223
|
-
summary: Extensions for working with
|
282
|
+
summary: Extensions to Aws::Record for working with DynamoDB.
|
224
283
|
test_files: []
|