dynamo-autoscale 0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (52) hide show
  1. data/.gitignore +4 -0
  2. data/Gemfile +13 -0
  3. data/Gemfile.lock +58 -0
  4. data/LICENSE +21 -0
  5. data/README.md +400 -0
  6. data/Rakefile +9 -0
  7. data/aws.sample.yml +16 -0
  8. data/bin/dynamo-autoscale +131 -0
  9. data/config/environment/common.rb +114 -0
  10. data/config/environment/console.rb +2 -0
  11. data/config/environment/test.rb +3 -0
  12. data/config/logger.yml +11 -0
  13. data/config/services/aws.rb +20 -0
  14. data/config/services/logger.rb +35 -0
  15. data/data/.gitkeep +0 -0
  16. data/dynamo-autoscale.gemspec +29 -0
  17. data/lib/dynamo-autoscale/actioner.rb +265 -0
  18. data/lib/dynamo-autoscale/cw_poller.rb +49 -0
  19. data/lib/dynamo-autoscale/dispatcher.rb +39 -0
  20. data/lib/dynamo-autoscale/dynamo_actioner.rb +59 -0
  21. data/lib/dynamo-autoscale/ext/active_support/duration.rb +7 -0
  22. data/lib/dynamo-autoscale/local_actioner.rb +39 -0
  23. data/lib/dynamo-autoscale/local_data_poll.rb +51 -0
  24. data/lib/dynamo-autoscale/logger.rb +15 -0
  25. data/lib/dynamo-autoscale/metrics.rb +192 -0
  26. data/lib/dynamo-autoscale/poller.rb +41 -0
  27. data/lib/dynamo-autoscale/pretty_formatter.rb +27 -0
  28. data/lib/dynamo-autoscale/rule.rb +180 -0
  29. data/lib/dynamo-autoscale/rule_set.rb +69 -0
  30. data/lib/dynamo-autoscale/table_tracker.rb +329 -0
  31. data/lib/dynamo-autoscale/unit_cost.rb +41 -0
  32. data/lib/dynamo-autoscale/version.rb +3 -0
  33. data/lib/dynamo-autoscale.rb +1 -0
  34. data/rlib/dynamodb_graph.r +15 -0
  35. data/rlib/dynamodb_scatterplot.r +13 -0
  36. data/rulesets/default.rb +5 -0
  37. data/rulesets/erroneous.rb +1 -0
  38. data/rulesets/gradual_tail.rb +11 -0
  39. data/rulesets/none.rb +0 -0
  40. data/script/console +3 -0
  41. data/script/historic_data +46 -0
  42. data/script/hourly_wastage +40 -0
  43. data/script/monitor +55 -0
  44. data/script/simulator +40 -0
  45. data/script/test +52 -0
  46. data/script/validate_ruleset +20 -0
  47. data/spec/actioner_spec.rb +244 -0
  48. data/spec/rule_set_spec.rb +89 -0
  49. data/spec/rule_spec.rb +491 -0
  50. data/spec/spec_helper.rb +4 -0
  51. data/spec/table_tracker_spec.rb +256 -0
  52. metadata +178 -0
@@ -0,0 +1,244 @@
1
+ require 'spec_helper'
2
+
3
+ describe DynamoAutoscale::Actioner do
4
+ let(:table) { DynamoAutoscale::TableTracker.new("table") }
5
+ let(:actioner) { DynamoAutoscale::LocalActioner.new(table) }
6
+
7
+ before { DynamoAutoscale.current_table = table }
8
+ after { DynamoAutoscale.current_table = nil }
9
+ after { Timecop.return }
10
+
11
+ describe "scaling down" do
12
+ before do
13
+ table.tick(5.minutes.ago, {
14
+ provisioned_writes: 15000, consumed_writes: 50,
15
+ provisioned_reads: 15000, consumed_reads: 20,
16
+ })
17
+ end
18
+
19
+ it "should not be allowed more than 4 times per day" do
20
+ actioner.set(:writes, 90).should be_true
21
+ Timecop.travel(10.minutes.from_now)
22
+ actioner.set(:writes, 80).should be_true
23
+ Timecop.travel(10.minutes.from_now)
24
+ actioner.set(:writes, 70).should be_true
25
+ Timecop.travel(10.minutes.from_now)
26
+ actioner.set(:writes, 60).should be_true
27
+ Timecop.travel(10.minutes.from_now)
28
+ actioner.set(:writes, 60).should be_false
29
+ end
30
+
31
+ it "is not per metric, it is per table" do
32
+ actioner.set(:reads, 90).should be_true
33
+ Timecop.travel(10.minutes.from_now)
34
+ actioner.set(:writes, 80).should be_true
35
+ Timecop.travel(10.minutes.from_now)
36
+ actioner.set(:reads, 70).should be_true
37
+ Timecop.travel(10.minutes.from_now)
38
+ actioner.set(:writes, 60).should be_true
39
+ Timecop.travel(10.minutes.from_now)
40
+ actioner.set(:writes, 60).should be_false
41
+ end
42
+
43
+ it "should not be allowed to fall below the minimum throughput" do
44
+ actioner.set(:reads, DynamoAutoscale::Actioner.minimum_throughput - 1)
45
+ time, val = actioner.provisioned_reads.last
46
+ val.should == DynamoAutoscale::Actioner.minimum_throughput
47
+ end
48
+
49
+ it "should not be allowed to go above the maximum throughput" do
50
+ actioner.set(:reads, DynamoAutoscale::Actioner.maximum_throughput + 1)
51
+ time, val = actioner.provisioned_reads.last
52
+ val.should == DynamoAutoscale::Actioner.maximum_throughput
53
+ end
54
+ end
55
+
56
+ describe "scale resets" do
57
+ before do
58
+ table.tick(5.minutes.ago, {
59
+ provisioned_writes: 100, consumed_writes: 50,
60
+ provisioned_reads: 100, consumed_reads: 20,
61
+ })
62
+ end
63
+
64
+ it "once per day at midnight" do
65
+ Timecop.travel(1.day.from_now.utc.midnight - 6.hours)
66
+
67
+ Timecop.travel(10.minutes.from_now)
68
+ actioner.set(:writes, 90)
69
+ Timecop.travel(10.minutes.from_now)
70
+ actioner.set(:writes, 80)
71
+ Timecop.travel(10.minutes.from_now)
72
+ actioner.set(:writes, 70)
73
+ Timecop.travel(10.minutes.from_now)
74
+ actioner.set(:writes, 60)
75
+ Timecop.travel(10.minutes.from_now)
76
+ actioner.set(:writes, 50)
77
+
78
+ actioner.provisioned_writes.length.should == 4
79
+ actioner.downscales.should == 4
80
+ actioner.upscales.should == 0
81
+ time, value = actioner.provisioned_for(:writes).last
82
+ value.should == 60
83
+
84
+ Timecop.travel(1.day.from_now.utc.midnight)
85
+
86
+ actioner.set(:writes, 50)
87
+ Timecop.travel(10.minutes.from_now)
88
+ actioner.set(:writes, 40)
89
+ Timecop.travel(10.minutes.from_now)
90
+ actioner.set(:writes, 30)
91
+ Timecop.travel(10.minutes.from_now)
92
+ actioner.set(:writes, 20)
93
+ Timecop.travel(10.minutes.from_now)
94
+ actioner.set(:writes, 10)
95
+
96
+ actioner.provisioned_writes.length.should == 8
97
+ actioner.downscales.should == 4
98
+ actioner.upscales.should == 0
99
+ time, value = actioner.provisioned_for(:writes).last
100
+ value.should == 20
101
+ end
102
+
103
+ specify "and not a second sooner" do
104
+ actioner.set(:writes, 90).should be_true
105
+ Timecop.travel(10.minutes.from_now)
106
+ actioner.set(:writes, 80).should be_true
107
+ Timecop.travel(10.minutes.from_now)
108
+ actioner.set(:writes, 70).should be_true
109
+ Timecop.travel(10.minutes.from_now)
110
+ actioner.set(:writes, 60).should be_true
111
+ Timecop.travel(10.minutes.from_now)
112
+ actioner.set(:writes, 60).should be_false
113
+ actioner.downscales.should == 4
114
+ actioner.upscales.should == 0
115
+
116
+ Timecop.travel(1.day.from_now.utc.midnight - 1.second)
117
+
118
+ actioner.set(:writes, 50).should be_false
119
+ actioner.downscales.should == 4
120
+ actioner.upscales.should == 0
121
+ end
122
+ end
123
+
124
+ describe "scaling up" do
125
+ before do
126
+ table.tick(5.minutes.ago, {
127
+ provisioned_writes: 100, consumed_writes: 50,
128
+ provisioned_reads: 100, consumed_reads: 20,
129
+ })
130
+
131
+ actioner.set(:writes, 100000).should be_true
132
+ end
133
+
134
+ it "should only go up to 2x your current provisioned" do
135
+ time, val = actioner.provisioned_writes.last
136
+ val.should == 200
137
+ end
138
+
139
+ it "can happen as much as it fucking wants to" do
140
+ Timecop.travel(10.minutes.from_now)
141
+ actioner.set(:writes, 200).should be_true
142
+ Timecop.travel(10.minutes.from_now)
143
+ actioner.set(:writes, 300).should be_true
144
+ Timecop.travel(10.minutes.from_now)
145
+ actioner.set(:writes, 400).should be_true
146
+ Timecop.travel(10.minutes.from_now)
147
+ actioner.set(:writes, 500).should be_true
148
+ Timecop.travel(10.minutes.from_now)
149
+ actioner.set(:writes, 600).should be_true
150
+ Timecop.travel(10.minutes.from_now)
151
+ actioner.set(:writes, 700).should be_true
152
+ Timecop.travel(10.minutes.from_now)
153
+ actioner.set(:writes, 800).should be_true
154
+ Timecop.travel(10.minutes.from_now)
155
+ actioner.set(:writes, 900).should be_true
156
+ Timecop.travel(10.minutes.from_now)
157
+ end
158
+ end
159
+
160
+ describe "grouping actions" do
161
+ let(:actioner) { DynamoAutoscale::LocalActioner.new(table, group_downscales: true) }
162
+
163
+ before do
164
+ table.tick(5.minutes.ago, {
165
+ provisioned_writes: 100, consumed_writes: 50,
166
+ provisioned_reads: 100, consumed_reads: 20,
167
+ })
168
+ end
169
+
170
+ describe "writes" do
171
+ before do
172
+ actioner.set(:writes, 10)
173
+ end
174
+
175
+ it "should not apply a write without an accompanying read" do
176
+ actioner.provisioned_for(:writes).last.should be_nil
177
+ end
178
+ end
179
+
180
+ describe "reads" do
181
+ before do
182
+ actioner.set(:reads, 10)
183
+ end
184
+
185
+ it "should not apply a read without an accompanying write" do
186
+ actioner.provisioned_for(:reads).last.should be_nil
187
+ end
188
+ end
189
+
190
+ describe "a write and a read" do
191
+ before do
192
+ actioner.set(:reads, 30)
193
+ actioner.set(:writes, 30)
194
+ end
195
+
196
+ it "should be applied" do
197
+ time, value = actioner.provisioned_for(:reads).last
198
+ value.should == 30
199
+
200
+ time, value = actioner.provisioned_for(:writes).last
201
+ value.should == 30
202
+ end
203
+ end
204
+
205
+ describe "flushing after a period of time" do
206
+ let(:actioner) do
207
+ DynamoAutoscale::LocalActioner.new(table, {
208
+ group_downscales: true,
209
+ flush_after: 5.minutes,
210
+ })
211
+ end
212
+
213
+ describe "happy path" do
214
+ before do
215
+ actioner.set(:reads, 20)
216
+ actioner.set(:reads, 10)
217
+
218
+ Timecop.travel(10.minutes.from_now)
219
+ actioner.try_flush!
220
+ end
221
+
222
+ it "should flush" do
223
+ actioner.provisioned_reads.length.should == 1
224
+ time, value = actioner.provisioned_reads.last
225
+ value.should == 10
226
+ end
227
+ end
228
+
229
+ describe "unhappy path" do
230
+ before do
231
+ actioner.set(:reads, 20)
232
+ actioner.set(:reads, 10)
233
+ actioner.try_flush!
234
+ end
235
+
236
+ it "should not flush" do
237
+ actioner.provisioned_reads.length.should == 0
238
+ time, value = actioner.provisioned_reads.last
239
+ value.should be_nil
240
+ end
241
+ end
242
+ end
243
+ end
244
+ end
@@ -0,0 +1,89 @@
1
+ require 'spec_helper'
2
+
3
+ describe DynamoAutoscale::RuleSet do
4
+ describe 'creating rules' do
5
+ let :rules do
6
+ DynamoAutoscale::RuleSet.new do
7
+ table "test" do
8
+ reads greater_than: 50, for: 5.minutes do
9
+
10
+ end
11
+
12
+ writes greater_than: 100, for: 15.minutes do
13
+
14
+ end
15
+ end
16
+
17
+ table :all do
18
+ reads less_than: 20, for: 2 do
19
+
20
+ end
21
+ end
22
+
23
+ writes greater_than: "40%", for: 12.seconds do
24
+
25
+ end
26
+ end
27
+ end
28
+
29
+ describe 'for a single table' do
30
+ subject { rules.for "test" }
31
+ its(:length) { should == 4 }
32
+ end
33
+
34
+ describe 'for all tables' do
35
+ subject { rules.for :all }
36
+ its(:length) { should == 2 }
37
+ end
38
+ end
39
+
40
+ describe 'using rules' do
41
+ let :rules do
42
+ DynamoAutoscale::RuleSet.new do
43
+ table "test_table" do
44
+ reads greater_than: 50, for: 5.minutes do
45
+ @__first = true
46
+ end
47
+
48
+ reads greater_than: 100, for: 15.minutes do
49
+ @__second = true
50
+ end
51
+ end
52
+
53
+ reads greater_than: "40%", for: 12.minutes do
54
+ @__third = true
55
+ end
56
+ end
57
+ end
58
+
59
+ describe 'earlier rules get precedence' do
60
+ let(:table) { DynamoAutoscale::TableTracker.new("test_table") }
61
+
62
+ before do
63
+ table.tick(4.minutes.ago, {
64
+ provisioned_reads: 100.0,
65
+ provisioned_writes: 200.0,
66
+ consumed_reads: 90.0,
67
+ consumed_writes: 30.0,
68
+ })
69
+
70
+ rules.test(table)
71
+ end
72
+
73
+ describe 'first block should get called' do
74
+ subject { rules.instance_variable_get(:@__first) }
75
+ it { should be_true }
76
+ end
77
+
78
+ describe 'second block should not get called' do
79
+ subject { rules.instance_variable_get(:@__second) }
80
+ it { should be_nil }
81
+ end
82
+
83
+ describe 'third block should not get called' do
84
+ subject { rules.instance_variable_get(:@__third) }
85
+ it { should be_nil }
86
+ end
87
+ end
88
+ end
89
+ end