dynamo-autoscale 0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. data/.gitignore +4 -0
  2. data/Gemfile +13 -0
  3. data/Gemfile.lock +58 -0
  4. data/LICENSE +21 -0
  5. data/README.md +400 -0
  6. data/Rakefile +9 -0
  7. data/aws.sample.yml +16 -0
  8. data/bin/dynamo-autoscale +131 -0
  9. data/config/environment/common.rb +114 -0
  10. data/config/environment/console.rb +2 -0
  11. data/config/environment/test.rb +3 -0
  12. data/config/logger.yml +11 -0
  13. data/config/services/aws.rb +20 -0
  14. data/config/services/logger.rb +35 -0
  15. data/data/.gitkeep +0 -0
  16. data/dynamo-autoscale.gemspec +29 -0
  17. data/lib/dynamo-autoscale/actioner.rb +265 -0
  18. data/lib/dynamo-autoscale/cw_poller.rb +49 -0
  19. data/lib/dynamo-autoscale/dispatcher.rb +39 -0
  20. data/lib/dynamo-autoscale/dynamo_actioner.rb +59 -0
  21. data/lib/dynamo-autoscale/ext/active_support/duration.rb +7 -0
  22. data/lib/dynamo-autoscale/local_actioner.rb +39 -0
  23. data/lib/dynamo-autoscale/local_data_poll.rb +51 -0
  24. data/lib/dynamo-autoscale/logger.rb +15 -0
  25. data/lib/dynamo-autoscale/metrics.rb +192 -0
  26. data/lib/dynamo-autoscale/poller.rb +41 -0
  27. data/lib/dynamo-autoscale/pretty_formatter.rb +27 -0
  28. data/lib/dynamo-autoscale/rule.rb +180 -0
  29. data/lib/dynamo-autoscale/rule_set.rb +69 -0
  30. data/lib/dynamo-autoscale/table_tracker.rb +329 -0
  31. data/lib/dynamo-autoscale/unit_cost.rb +41 -0
  32. data/lib/dynamo-autoscale/version.rb +3 -0
  33. data/lib/dynamo-autoscale.rb +1 -0
  34. data/rlib/dynamodb_graph.r +15 -0
  35. data/rlib/dynamodb_scatterplot.r +13 -0
  36. data/rulesets/default.rb +5 -0
  37. data/rulesets/erroneous.rb +1 -0
  38. data/rulesets/gradual_tail.rb +11 -0
  39. data/rulesets/none.rb +0 -0
  40. data/script/console +3 -0
  41. data/script/historic_data +46 -0
  42. data/script/hourly_wastage +40 -0
  43. data/script/monitor +55 -0
  44. data/script/simulator +40 -0
  45. data/script/test +52 -0
  46. data/script/validate_ruleset +20 -0
  47. data/spec/actioner_spec.rb +244 -0
  48. data/spec/rule_set_spec.rb +89 -0
  49. data/spec/rule_spec.rb +491 -0
  50. data/spec/spec_helper.rb +4 -0
  51. data/spec/table_tracker_spec.rb +256 -0
  52. metadata +178 -0
@@ -0,0 +1,244 @@
1
+ require 'spec_helper'
2
+
3
+ describe DynamoAutoscale::Actioner do
4
+ let(:table) { DynamoAutoscale::TableTracker.new("table") }
5
+ let(:actioner) { DynamoAutoscale::LocalActioner.new(table) }
6
+
7
+ before { DynamoAutoscale.current_table = table }
8
+ after { DynamoAutoscale.current_table = nil }
9
+ after { Timecop.return }
10
+
11
+ describe "scaling down" do
12
+ before do
13
+ table.tick(5.minutes.ago, {
14
+ provisioned_writes: 15000, consumed_writes: 50,
15
+ provisioned_reads: 15000, consumed_reads: 20,
16
+ })
17
+ end
18
+
19
+ it "should not be allowed more than 4 times per day" do
20
+ actioner.set(:writes, 90).should be_true
21
+ Timecop.travel(10.minutes.from_now)
22
+ actioner.set(:writes, 80).should be_true
23
+ Timecop.travel(10.minutes.from_now)
24
+ actioner.set(:writes, 70).should be_true
25
+ Timecop.travel(10.minutes.from_now)
26
+ actioner.set(:writes, 60).should be_true
27
+ Timecop.travel(10.minutes.from_now)
28
+ actioner.set(:writes, 60).should be_false
29
+ end
30
+
31
+ it "is not per metric, it is per table" do
32
+ actioner.set(:reads, 90).should be_true
33
+ Timecop.travel(10.minutes.from_now)
34
+ actioner.set(:writes, 80).should be_true
35
+ Timecop.travel(10.minutes.from_now)
36
+ actioner.set(:reads, 70).should be_true
37
+ Timecop.travel(10.minutes.from_now)
38
+ actioner.set(:writes, 60).should be_true
39
+ Timecop.travel(10.minutes.from_now)
40
+ actioner.set(:writes, 60).should be_false
41
+ end
42
+
43
+ it "should not be allowed to fall below the minimum throughput" do
44
+ actioner.set(:reads, DynamoAutoscale::Actioner.minimum_throughput - 1)
45
+ time, val = actioner.provisioned_reads.last
46
+ val.should == DynamoAutoscale::Actioner.minimum_throughput
47
+ end
48
+
49
+ it "should not be allowed to go above the maximum throughput" do
50
+ actioner.set(:reads, DynamoAutoscale::Actioner.maximum_throughput + 1)
51
+ time, val = actioner.provisioned_reads.last
52
+ val.should == DynamoAutoscale::Actioner.maximum_throughput
53
+ end
54
+ end
55
+
56
+ describe "scale resets" do
57
+ before do
58
+ table.tick(5.minutes.ago, {
59
+ provisioned_writes: 100, consumed_writes: 50,
60
+ provisioned_reads: 100, consumed_reads: 20,
61
+ })
62
+ end
63
+
64
+ it "once per day at midnight" do
65
+ Timecop.travel(1.day.from_now.utc.midnight - 6.hours)
66
+
67
+ Timecop.travel(10.minutes.from_now)
68
+ actioner.set(:writes, 90)
69
+ Timecop.travel(10.minutes.from_now)
70
+ actioner.set(:writes, 80)
71
+ Timecop.travel(10.minutes.from_now)
72
+ actioner.set(:writes, 70)
73
+ Timecop.travel(10.minutes.from_now)
74
+ actioner.set(:writes, 60)
75
+ Timecop.travel(10.minutes.from_now)
76
+ actioner.set(:writes, 50)
77
+
78
+ actioner.provisioned_writes.length.should == 4
79
+ actioner.downscales.should == 4
80
+ actioner.upscales.should == 0
81
+ time, value = actioner.provisioned_for(:writes).last
82
+ value.should == 60
83
+
84
+ Timecop.travel(1.day.from_now.utc.midnight)
85
+
86
+ actioner.set(:writes, 50)
87
+ Timecop.travel(10.minutes.from_now)
88
+ actioner.set(:writes, 40)
89
+ Timecop.travel(10.minutes.from_now)
90
+ actioner.set(:writes, 30)
91
+ Timecop.travel(10.minutes.from_now)
92
+ actioner.set(:writes, 20)
93
+ Timecop.travel(10.minutes.from_now)
94
+ actioner.set(:writes, 10)
95
+
96
+ actioner.provisioned_writes.length.should == 8
97
+ actioner.downscales.should == 4
98
+ actioner.upscales.should == 0
99
+ time, value = actioner.provisioned_for(:writes).last
100
+ value.should == 20
101
+ end
102
+
103
+ specify "and not a second sooner" do
104
+ actioner.set(:writes, 90).should be_true
105
+ Timecop.travel(10.minutes.from_now)
106
+ actioner.set(:writes, 80).should be_true
107
+ Timecop.travel(10.minutes.from_now)
108
+ actioner.set(:writes, 70).should be_true
109
+ Timecop.travel(10.minutes.from_now)
110
+ actioner.set(:writes, 60).should be_true
111
+ Timecop.travel(10.minutes.from_now)
112
+ actioner.set(:writes, 60).should be_false
113
+ actioner.downscales.should == 4
114
+ actioner.upscales.should == 0
115
+
116
+ Timecop.travel(1.day.from_now.utc.midnight - 1.second)
117
+
118
+ actioner.set(:writes, 50).should be_false
119
+ actioner.downscales.should == 4
120
+ actioner.upscales.should == 0
121
+ end
122
+ end
123
+
124
+ describe "scaling up" do
125
+ before do
126
+ table.tick(5.minutes.ago, {
127
+ provisioned_writes: 100, consumed_writes: 50,
128
+ provisioned_reads: 100, consumed_reads: 20,
129
+ })
130
+
131
+ actioner.set(:writes, 100000).should be_true
132
+ end
133
+
134
+ it "should only go up to 2x your current provisioned" do
135
+ time, val = actioner.provisioned_writes.last
136
+ val.should == 200
137
+ end
138
+
139
+ it "can happen as much as it fucking wants to" do
140
+ Timecop.travel(10.minutes.from_now)
141
+ actioner.set(:writes, 200).should be_true
142
+ Timecop.travel(10.minutes.from_now)
143
+ actioner.set(:writes, 300).should be_true
144
+ Timecop.travel(10.minutes.from_now)
145
+ actioner.set(:writes, 400).should be_true
146
+ Timecop.travel(10.minutes.from_now)
147
+ actioner.set(:writes, 500).should be_true
148
+ Timecop.travel(10.minutes.from_now)
149
+ actioner.set(:writes, 600).should be_true
150
+ Timecop.travel(10.minutes.from_now)
151
+ actioner.set(:writes, 700).should be_true
152
+ Timecop.travel(10.minutes.from_now)
153
+ actioner.set(:writes, 800).should be_true
154
+ Timecop.travel(10.minutes.from_now)
155
+ actioner.set(:writes, 900).should be_true
156
+ Timecop.travel(10.minutes.from_now)
157
+ end
158
+ end
159
+
160
+ describe "grouping actions" do
161
+ let(:actioner) { DynamoAutoscale::LocalActioner.new(table, group_downscales: true) }
162
+
163
+ before do
164
+ table.tick(5.minutes.ago, {
165
+ provisioned_writes: 100, consumed_writes: 50,
166
+ provisioned_reads: 100, consumed_reads: 20,
167
+ })
168
+ end
169
+
170
+ describe "writes" do
171
+ before do
172
+ actioner.set(:writes, 10)
173
+ end
174
+
175
+ it "should not apply a write without an accompanying read" do
176
+ actioner.provisioned_for(:writes).last.should be_nil
177
+ end
178
+ end
179
+
180
+ describe "reads" do
181
+ before do
182
+ actioner.set(:reads, 10)
183
+ end
184
+
185
+ it "should not apply a read without an accompanying write" do
186
+ actioner.provisioned_for(:reads).last.should be_nil
187
+ end
188
+ end
189
+
190
+ describe "a write and a read" do
191
+ before do
192
+ actioner.set(:reads, 30)
193
+ actioner.set(:writes, 30)
194
+ end
195
+
196
+ it "should be applied" do
197
+ time, value = actioner.provisioned_for(:reads).last
198
+ value.should == 30
199
+
200
+ time, value = actioner.provisioned_for(:writes).last
201
+ value.should == 30
202
+ end
203
+ end
204
+
205
+ describe "flushing after a period of time" do
206
+ let(:actioner) do
207
+ DynamoAutoscale::LocalActioner.new(table, {
208
+ group_downscales: true,
209
+ flush_after: 5.minutes,
210
+ })
211
+ end
212
+
213
+ describe "happy path" do
214
+ before do
215
+ actioner.set(:reads, 20)
216
+ actioner.set(:reads, 10)
217
+
218
+ Timecop.travel(10.minutes.from_now)
219
+ actioner.try_flush!
220
+ end
221
+
222
+ it "should flush" do
223
+ actioner.provisioned_reads.length.should == 1
224
+ time, value = actioner.provisioned_reads.last
225
+ value.should == 10
226
+ end
227
+ end
228
+
229
+ describe "unhappy path" do
230
+ before do
231
+ actioner.set(:reads, 20)
232
+ actioner.set(:reads, 10)
233
+ actioner.try_flush!
234
+ end
235
+
236
+ it "should not flush" do
237
+ actioner.provisioned_reads.length.should == 0
238
+ time, value = actioner.provisioned_reads.last
239
+ value.should be_nil
240
+ end
241
+ end
242
+ end
243
+ end
244
+ end
@@ -0,0 +1,89 @@
1
+ require 'spec_helper'
2
+
3
+ describe DynamoAutoscale::RuleSet do
4
+ describe 'creating rules' do
5
+ let :rules do
6
+ DynamoAutoscale::RuleSet.new do
7
+ table "test" do
8
+ reads greater_than: 50, for: 5.minutes do
9
+
10
+ end
11
+
12
+ writes greater_than: 100, for: 15.minutes do
13
+
14
+ end
15
+ end
16
+
17
+ table :all do
18
+ reads less_than: 20, for: 2 do
19
+
20
+ end
21
+ end
22
+
23
+ writes greater_than: "40%", for: 12.seconds do
24
+
25
+ end
26
+ end
27
+ end
28
+
29
+ describe 'for a single table' do
30
+ subject { rules.for "test" }
31
+ its(:length) { should == 4 }
32
+ end
33
+
34
+ describe 'for all tables' do
35
+ subject { rules.for :all }
36
+ its(:length) { should == 2 }
37
+ end
38
+ end
39
+
40
+ describe 'using rules' do
41
+ let :rules do
42
+ DynamoAutoscale::RuleSet.new do
43
+ table "test_table" do
44
+ reads greater_than: 50, for: 5.minutes do
45
+ @__first = true
46
+ end
47
+
48
+ reads greater_than: 100, for: 15.minutes do
49
+ @__second = true
50
+ end
51
+ end
52
+
53
+ reads greater_than: "40%", for: 12.minutes do
54
+ @__third = true
55
+ end
56
+ end
57
+ end
58
+
59
+ describe 'earlier rules get precedence' do
60
+ let(:table) { DynamoAutoscale::TableTracker.new("test_table") }
61
+
62
+ before do
63
+ table.tick(4.minutes.ago, {
64
+ provisioned_reads: 100.0,
65
+ provisioned_writes: 200.0,
66
+ consumed_reads: 90.0,
67
+ consumed_writes: 30.0,
68
+ })
69
+
70
+ rules.test(table)
71
+ end
72
+
73
+ describe 'first block should get called' do
74
+ subject { rules.instance_variable_get(:@__first) }
75
+ it { should be_true }
76
+ end
77
+
78
+ describe 'second block should not get called' do
79
+ subject { rules.instance_variable_get(:@__second) }
80
+ it { should be_nil }
81
+ end
82
+
83
+ describe 'third block should not get called' do
84
+ subject { rules.instance_variable_get(:@__third) }
85
+ it { should be_nil }
86
+ end
87
+ end
88
+ end
89
+ end