vector 0.0.4 → 0.0.5

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 541f21bb927907e0a40dc80e6383fafadbfed41b
4
+ data.tar.gz: 608fb28e14b4f8b6e35ae093ef30d02346b910ca
5
+ SHA512:
6
+ metadata.gz: e54156a992bdec104878d7be100673797e153198d3f9888307df09966421ec7c63833529745a4c61cba22c905f9df8b7f8c9aa56bd1900c510b12ebe97a6b30e
7
+ data.tar.gz: 272cdb7eba79cdc60425fb2c14dbbb793911696b62c401e695f97beeacd2f48b198e1b25e48e082013bdd747578378b532f737d8e04f929be76892de760fddfb
data/README.md CHANGED
@@ -123,6 +123,21 @@ renewals - the group will not be scaled down.
123
123
  one, or else it's possible Vector may never find eligible nodes for
124
124
  scaledown and never scaledown.)
125
125
 
126
+ ### Variable Thresholds
127
+
128
+ When deciding to scale down, a static CPU utilization threshold may be
129
+ inefficient. For example, if there are 2 nodes running, and you have a
130
+ minimum of 2, and the average CPU is 75%, removing 1 node would
131
+ theoretically result in the remaining 2 nodes running at > 100%. However,
132
+ with 20 nodes running, at an average CPU of 75%, removing 1 node will
133
+ only result in an average CPU of 79% across the remaining 19 nodes.
134
+
135
+ When there are more nodes running, you can be more aggressive about
136
+ removing nodes without overloading the remaining nodes. Variable
137
+ thresholds allow you to express this.
138
+
139
+ You can enable variable thresholds with `--fds-variable-thresholds`.
140
+
126
141
  ### Integration with Predictive Scaling
127
142
 
128
143
  Before scaling down, and if Predictive Scaling is in effect, Vector will
@@ -32,14 +32,14 @@ module Vector
32
32
  if @config[:predictive_scaling][:enabled]
33
33
  psconf = @config[:predictive_scaling]
34
34
  ps = Vector::Function::PredictiveScaling.new(
35
- { :cloudwatch => cloudwatch }.merge(psconf))
35
+ { :cloudwatch => cloudwatch, :dry_run => @config[:dry_run] }.merge(psconf))
36
36
  end
37
37
 
38
38
  fds = nil
39
39
  if @config[:flexible_down_scaling][:enabled]
40
40
  fdsconf = @config[:flexible_down_scaling]
41
41
  fds = Vector::Function::FlexibleDownScaling.new(
42
- { :cloudwatch => cloudwatch }.merge(fdsconf))
42
+ { :cloudwatch => cloudwatch, :dry_run => @config[:dry_run] }.merge(fdsconf))
43
43
  end
44
44
 
45
45
  groups.each do |group|
@@ -71,6 +71,7 @@ module Vector
71
71
  def load_config
72
72
  opts = {
73
73
  :quiet => false,
74
+ :dry_run => false,
74
75
  :region => 'us-east-1',
75
76
  :groups => [],
76
77
  :fleet => nil,
@@ -85,56 +86,72 @@ module Vector
85
86
  :enabled => false,
86
87
  :up_down_cooldown => nil,
87
88
  :down_down_cooldown => nil,
88
- :max_sunk_cost => nil
89
+ :max_sunk_cost => nil,
90
+ :variable_thresholds => false,
91
+ :n_low => nil,
92
+ :n_high => nil,
93
+ :m => nil,
94
+ :g_high => 1.0,
95
+ :g_low => 1.0
89
96
  }
90
97
  }
91
98
 
92
99
  optparser = OptionParser.new do |o|
93
100
  o.banner = "Usage: vector [options]"
94
101
  o.separator "DURATION can look like 60s, 1m, 5h, 7d, 1w"
102
+ o.set_summary_width 5
103
+ o.set_summary_indent ' '
95
104
 
96
- o.on("--timezone TIMEZONE", "Timezone to use for date calculations (like America/Denver) (default: system timezone)") do |v|
105
+ def wrap(str)
106
+ str.scan(/\S.{0,#{60}}\S(?=\s|$)|\S+/).join "\n "
107
+ end
108
+
109
+ o.on("--timezone TIMEZONE", wrap("Timezone to use for date calculations (like America/Denver) (default: system timezone)")) do |v|
97
110
  Time.zone = v
98
111
  end
99
112
 
100
- o.on("--region REGION", "AWS region to operate in (default: us-east-1)") do |v|
113
+ o.on("--region REGION", wrap("AWS region to operate in (default: us-east-1)")) do |v|
101
114
  opts[:region] = v
102
115
  end
103
116
 
104
- o.on("--groups group1,group2", Array, "A list of Auto Scaling Groups to evaluate") do |v|
117
+ o.on("--groups group1,group2", Array, wrap("A list of Auto Scaling Groups to evaluate")) do |v|
105
118
  opts[:groups] = v
106
119
  end
107
120
 
108
- o.on("--fleet fleet", "An AWS ASG Fleet (instead of specifying --groups)") do |v|
121
+ o.on("--fleet fleet", wrap("An AWS ASG Fleet (instead of specifying --groups)")) do |v|
109
122
  opts[:fleet] = v
110
123
  end
111
124
 
112
- o.on("-q", "--[no-]quiet", "Run quietly") do |v|
125
+ o.on("--[no-]dry-run", wrap("Don't actually trigger any policies")) do |v|
126
+ opts[:dry_run] = v
127
+ end
128
+
129
+ o.on("-q", "--[no-]quiet", wrap("Run quietly")) do |v|
113
130
  opts[:quiet] = v
114
131
  end
115
132
 
116
133
  o.separator ""
117
134
  o.separator "Predictive Scaling Options"
118
135
 
119
- o.on("--[no-]ps", "Enable Predictive Scaling") do |v|
136
+ o.on("--[no-]ps", wrap("Enable Predictive Scaling")) do |v|
120
137
  opts[:predictive_scaling][:enabled] = v
121
138
  end
122
139
 
123
- o.on("--ps-lookback-windows DURATION,DURATION", Array, "List of lookback windows") do |v|
140
+ o.on("--ps-lookback-windows DURATION,DURATION", Array, wrap("List of lookback windows")) do |v|
124
141
  opts[:predictive_scaling][:lookback_windows] =
125
142
  v.map {|w| Vector.time_string_to_seconds(w) }
126
143
  end
127
144
 
128
- o.on("--ps-lookahead-window DURATION", String, "Lookahead window") do |v|
145
+ o.on("--ps-lookahead-window DURATION", String, wrap("Lookahead window")) do |v|
129
146
  opts[:predictive_scaling][:lookahead_window] =
130
147
  Vector.time_string_to_seconds(v)
131
148
  end
132
149
 
133
- o.on("--ps-valid-threshold FLOAT", Float, "A number from 0.0 - 1.0 specifying how closely previous load must match current load for Predictive Scaling to take effect") do |v|
150
+ o.on("--ps-valid-threshold FLOAT", Float, wrap("A number from 0.0 - 1.0 specifying how closely previous load must match current load for Predictive Scaling to take effect")) do |v|
134
151
  opts[:predictive_scaling][:valid_threshold] = v
135
152
  end
136
153
 
137
- o.on("--ps-valid-period DURATION", String, "The period to use when doing the threshold check") do |v|
154
+ o.on("--ps-valid-period DURATION", String, wrap("The period to use when doing the threshold check")) do |v|
138
155
  opts[:predictive_scaling][:valid_period] =
139
156
  Vector.time_string_to_seconds v
140
157
  end
@@ -142,21 +159,21 @@ module Vector
142
159
  o.separator ""
143
160
  o.separator "Flexible Down Scaling Options"
144
161
 
145
- o.on("--[no-]fds", "Enable Flexible Down Scaling") do |v|
162
+ o.on("--[no-]fds", wrap("Enable Flexible Down Scaling")) do |v|
146
163
  opts[:flexible_down_scaling][:enabled] = v
147
164
  end
148
165
 
149
- o.on("--fds-up-to-down DURATION", String, "The cooldown period between up and down scale events") do |v|
166
+ o.on("--fds-up-to-down DURATION", String, wrap("The cooldown period between up and down scale events")) do |v|
150
167
  opts[:flexible_down_scaling][:up_down_cooldown] =
151
168
  Vector.time_string_to_seconds v
152
169
  end
153
170
 
154
- o.on("--fds-down-to-down DURATION", String, "The cooldown period between down and down scale events") do |v|
171
+ o.on("--fds-down-to-down DURATION", String, wrap("The cooldown period between down and down scale events")) do |v|
155
172
  opts[:flexible_down_scaling][:down_down_cooldown] =
156
173
  Vector.time_string_to_seconds v
157
174
  end
158
175
 
159
- o.on("--fds-max-sunk-cost DURATION", String, "Only let a scaledown occur if there is an instance this close to its hourly billing point") do |v|
176
+ o.on("--fds-max-sunk-cost DURATION", String, wrap("Only let a scaledown occur if there is an instance this close to its hourly billing point")) do |v|
160
177
  time = Vector.time_string_to_seconds v
161
178
  if time > 1.hour
162
179
  puts "--fds-max-sunk-cost duration must be < 1 hour"
@@ -166,6 +183,35 @@ module Vector
166
183
  opts[:flexible_down_scaling][:max_sunk_cost] = time
167
184
  end
168
185
 
186
+ o.separator ""
187
+ o.on("--[no-]fds-variable-thresholds", wrap("Enable Variable Thresholds")) do |v|
188
+ opts[:flexible_down_scaling][:variable_thresholds] = v
189
+ end
190
+
191
+ o.on("--fds-n-low NUM", Integer, wrap("Number of nodes corresponding to --fds-g-low. (default: 1 more than the group's minimum size)")) do |v|
192
+ opts[:flexible_down_scaling][:n_low] = v
193
+ end
194
+
195
+ o.on("--fds-n-high NUM", Integer, wrap("Number of nodes corresponding to --fds-g-high. (default: the group's maximum size)")) do |v|
196
+ opts[:flexible_down_scaling][:n_high] = v
197
+ end
198
+
199
+ o.on("--fds-m PERCENTAGE", Float, wrap("Maximum target utilization. Will default to the CPUUtilization alarm threshold.")) do |v|
200
+ opts[:flexible_down_scaling][:m] = v / 100
201
+ end
202
+
203
+ o.on("--fds-g-high PERCENTAGE", Float, wrap("Capacity headroom to apply when scaling down from --fds-n-high nodes, as a percentage. e.g. if this is 90%, then will not scale down from --fds-n-high nodes until expected utilization on the remaining nodes is at or below 90% of --fds-m. (default: 100)")) do |v|
204
+ opts[:flexible_down_scaling][:g_high] = v / 100
205
+ end
206
+
207
+ o.on("--fds-g-low PERCENTAGE", Float, wrap("Capacity headroom to apply when scaling down from --fds-n-low nodes, as a percentage. e.g. if this is 75%, then will not scale down from --fds-n-low nodes until expected utilization on the remaining nodes is at or below 75% of --fds-m. When scaling down from a number of nodes other than --fds-n-high or --fds-n-low, will use a capacity headroom linearly interpolated from --fds-g-high and --fds-g-low. (default: 100)")) do |v|
208
+ opts[:flexible_down_scaling][:g_low] = v / 100
209
+ end
210
+
211
+ o.on("--fds-print-variable-thresholds", wrap("Calculates and displays the thresholds that will be used for each asg, and does not execute any downscaling policies. (For debugging).")) do |v|
212
+ opts[:flexible_down_scaling][:print_variable_thresholds] = true
213
+ end
214
+
169
215
  end.parse!(@argv)
170
216
 
171
217
  if opts[:groups].empty? && opts[:fleet].nil?
@@ -7,9 +7,17 @@ module Vector
7
7
 
8
8
  def initialize(options)
9
9
  @cloudwatch = options[:cloudwatch]
10
+ @dry_run = options[:dry_run]
10
11
  @up_down_cooldown = options[:up_down_cooldown]
11
12
  @down_down_cooldown = options[:down_down_cooldown]
12
13
  @max_sunk_cost = options[:max_sunk_cost]
14
+ @variable_thresholds = options[:variable_thresholds]
15
+ @n_low = options[:n_low]
16
+ @n_high = options[:n_high]
17
+ @m = options[:m]
18
+ @g_low = options[:g_low]
19
+ @g_high = options[:g_high]
20
+ @debug_variable_thresholds = options[:print_variable_thresholds]
13
21
  end
14
22
 
15
23
  def run_for(group, ps_check_procs)
@@ -38,6 +46,7 @@ module Vector
38
46
  hlog_ctx("policy:#{policy.name}") do
39
47
  # TODO: support adjustment types other than ChangeInCapacity here
40
48
  if policy.adjustment_type == "ChangeInCapacity" &&
49
+ ps_check_procs &&
41
50
  ps_check_procs.any? {|ps_check_proc|
42
51
  ps_check_proc.call(group.desired_capacity + policy.scaling_adjustment) }
43
52
  hlog("Predictive scaleup would trigger a scaleup if group were shrunk")
@@ -54,11 +63,76 @@ module Vector
54
63
  !alarm.enabled?
55
64
  end
56
65
 
66
+ # Do this logic first in case the user is just trying to print out
67
+ # the thresholds.
68
+ if @variable_thresholds
69
+ # variable_thresholds currently requires a CPUUtilization alarm to function
70
+ vt_cpu_alarm = disabled_alarms.find {|alarm| alarm.metric_name == "CPUUtilization" }
71
+
72
+ # remove this alarm from the check, since we're not checking its alarm status
73
+ # below.
74
+ disabled_alarms.delete(vt_cpu_alarm)
75
+
76
+ unless vt_cpu_alarm
77
+ hlog("Variable thresholds requires an alarm on CPUUtilization, skipping")
78
+ next
79
+ end
80
+
81
+ @n_low ||= group.min_size + 1
82
+ @n_high ||= group.max_size
83
+ @m ||= vt_cpu_alarm.threshold / 100
84
+
85
+ if @g_low == @g_high
86
+ hlog("g_low == g_high (#{@g_low}), not attempting to use flexible thresholds.")
87
+ next
88
+ end
89
+
90
+ if @n_low == @n_high
91
+ hlog("n_low == n_high (#{@n_low}), not attempting to use flexible thresholds.")
92
+ next
93
+ end
94
+
95
+ if @debug_variable_thresholds
96
+ puts " n_low: #{@n_low}"
97
+ puts " n_high: #{@n_high}"
98
+ puts " m: #{@m}"
99
+ puts " g_low: #{@g_low}"
100
+ puts " g_high: #{@g_high}"
101
+ puts
102
+ puts " N Threshold"
103
+ ([@n_low, group.min_size].min + 1).upto([@n_high, group.max_size].max) do |i|
104
+ puts " %2d %.1f%%" % [i, (variable_threshold(i, @n_low, @n_high, @m, @g_low, @g_high) * 100)]
105
+ end
106
+ next
107
+ end
108
+ end
109
+
57
110
  unless disabled_alarms.all? {|alarm| alarm.state_value == "ALARM" }
58
111
  hlog("Not all alarms are in ALARM state")
59
112
  next
60
113
  end
61
114
 
115
+ if @variable_thresholds
116
+ threshold = variable_threshold(group.desired_capacity, @n_low, @n_high, @m, @g_low, @g_high)
117
+
118
+ stats = vt_cpu_alarm.metric.statistics(
119
+ :start_time => Time.now - (vt_cpu_alarm.period * vt_cpu_alarm.evaluation_periods),
120
+ :end_time => Time.now,
121
+ :statistics => [ vt_cpu_alarm.statistic ],
122
+ :period => vt_cpu_alarm.period)
123
+
124
+ if stats.datapoints.length < vt_cpu_alarm.evaluation_periods
125
+ hlog("Could not get enough datapoints for checking variable threshold");
126
+ next
127
+ end
128
+
129
+ if stats.datapoints.any? {|dp| dp[vt_cpu_alarm.statistic.downcase.to_sym] > (threshold * 100) }
130
+ hlog("Not all datapoints are beneath the variable threshold #{(threshold * 100).to_i}: #{stats.datapoints}")
131
+ next
132
+ end
133
+ end
134
+
135
+ outside_cooldown = outside_cooldown_period(group)
62
136
  unless outside_cooldown_period(group)
63
137
  hlog("Group is not outside the specified cooldown periods")
64
138
  next
@@ -69,8 +143,12 @@ module Vector
69
143
  next
70
144
  end
71
145
 
72
- hlog("Executing policy")
73
- policy.execute(:honor_cooldown => true)
146
+ if @dry_run
147
+ hlog("Executing policy (DRY RUN)")
148
+ else
149
+ hlog("Executing policy")
150
+ policy.execute(:honor_cooldown => true)
151
+ end
74
152
 
75
153
  result[:triggered] = true
76
154
 
@@ -86,6 +164,15 @@ module Vector
86
164
 
87
165
  protected
88
166
 
167
+ def variable_threshold(n, n_low, n_high, m, g_low, g_high)
168
+ m_high = g_high * m
169
+ m_low = g_low * m
170
+ a = (m_high - m_low).to_f / (n_high - n_low).to_f
171
+ b = m_low - (n_low * a)
172
+ res = (a * n + b) * (1.0 - (1.0 / n))
173
+ res
174
+ end
175
+
89
176
  def has_eligible_scaledown_instance(group)
90
177
  return true if @max_sunk_cost.nil?
91
178
 
@@ -5,6 +5,7 @@ module Vector
5
5
 
6
6
  def initialize(options)
7
7
  @cloudwatch = options[:cloudwatch]
8
+ @dry_run = options[:dry_run]
8
9
  @lookback_windows = options[:lookback_windows]
9
10
  @lookahead_window = options[:lookahead_window]
10
11
  @valid_threshold = options[:valid_threshold]
@@ -83,8 +84,12 @@ module Vector
83
84
  result[:check_procs] << check_proc
84
85
 
85
86
  if check_proc.call(now_num)
86
- hlog "Executing policy"
87
- policy.execute(honor_cooldown: true)
87
+ if @dry_run
88
+ hlog "Executing policy (DRY RUN)"
89
+ else
90
+ hlog "Executing policy"
91
+ policy.execute(honor_cooldown: true)
92
+ end
88
93
 
89
94
  result[:triggered] = true
90
95
 
@@ -1,3 +1,3 @@
1
1
  module Vector
2
- VERSION = "0.0.4"
2
+ VERSION = "0.0.5"
3
3
  end
metadata CHANGED
@@ -1,94 +1,83 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: vector
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
5
- prerelease:
4
+ version: 0.0.5
6
5
  platform: ruby
7
6
  authors:
8
7
  - Zach Wily
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2013-07-21 00:00:00.000000000 Z
11
+ date: 2015-01-13 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: aws-sdk
16
15
  requirement: !ruby/object:Gem::Requirement
17
- none: false
18
16
  requirements:
19
- - - ! '>='
17
+ - - ">="
20
18
  - !ruby/object:Gem::Version
21
19
  version: '0'
22
20
  type: :runtime
23
21
  prerelease: false
24
22
  version_requirements: !ruby/object:Gem::Requirement
25
- none: false
26
23
  requirements:
27
- - - ! '>='
24
+ - - ">="
28
25
  - !ruby/object:Gem::Version
29
26
  version: '0'
30
27
  - !ruby/object:Gem::Dependency
31
28
  name: aws-asg-fleet
32
29
  requirement: !ruby/object:Gem::Requirement
33
- none: false
34
30
  requirements:
35
- - - ! '>='
31
+ - - ">="
36
32
  - !ruby/object:Gem::Version
37
33
  version: '0'
38
34
  type: :runtime
39
35
  prerelease: false
40
36
  version_requirements: !ruby/object:Gem::Requirement
41
- none: false
42
37
  requirements:
43
- - - ! '>='
38
+ - - ">="
44
39
  - !ruby/object:Gem::Version
45
40
  version: '0'
46
41
  - !ruby/object:Gem::Dependency
47
42
  name: activesupport
48
43
  requirement: !ruby/object:Gem::Requirement
49
- none: false
50
44
  requirements:
51
- - - ! '>='
45
+ - - ">="
52
46
  - !ruby/object:Gem::Version
53
47
  version: '0'
54
48
  type: :runtime
55
49
  prerelease: false
56
50
  version_requirements: !ruby/object:Gem::Requirement
57
- none: false
58
51
  requirements:
59
- - - ! '>='
52
+ - - ">="
60
53
  - !ruby/object:Gem::Version
61
54
  version: '0'
62
55
  - !ruby/object:Gem::Dependency
63
56
  name: bundler
64
57
  requirement: !ruby/object:Gem::Requirement
65
- none: false
66
58
  requirements:
67
- - - ~>
59
+ - - "~>"
68
60
  - !ruby/object:Gem::Version
69
61
  version: '1.3'
70
62
  type: :development
71
63
  prerelease: false
72
64
  version_requirements: !ruby/object:Gem::Requirement
73
- none: false
74
65
  requirements:
75
- - - ~>
66
+ - - "~>"
76
67
  - !ruby/object:Gem::Version
77
68
  version: '1.3'
78
69
  - !ruby/object:Gem::Dependency
79
70
  name: rake
80
71
  requirement: !ruby/object:Gem::Requirement
81
- none: false
82
72
  requirements:
83
- - - ! '>='
73
+ - - ">="
84
74
  - !ruby/object:Gem::Version
85
75
  version: '0'
86
76
  type: :development
87
77
  prerelease: false
88
78
  version_requirements: !ruby/object:Gem::Requirement
89
- none: false
90
79
  requirements:
91
- - - ! '>='
80
+ - - ">="
92
81
  - !ruby/object:Gem::Version
93
82
  version: '0'
94
83
  description:
@@ -99,7 +88,7 @@ executables:
99
88
  extensions: []
100
89
  extra_rdoc_files: []
101
90
  files:
102
- - .gitignore
91
+ - ".gitignore"
103
92
  - Gemfile
104
93
  - LICENSE.txt
105
94
  - README.md
@@ -114,27 +103,26 @@ files:
114
103
  homepage: http://github.com/instructure/vector
115
104
  licenses:
116
105
  - MIT
106
+ metadata: {}
117
107
  post_install_message:
118
108
  rdoc_options: []
119
109
  require_paths:
120
110
  - lib
121
111
  required_ruby_version: !ruby/object:Gem::Requirement
122
- none: false
123
112
  requirements:
124
- - - ! '>='
113
+ - - ">="
125
114
  - !ruby/object:Gem::Version
126
115
  version: '0'
127
116
  required_rubygems_version: !ruby/object:Gem::Requirement
128
- none: false
129
117
  requirements:
130
- - - ! '>='
118
+ - - ">="
131
119
  - !ruby/object:Gem::Version
132
120
  version: '0'
133
121
  requirements: []
134
122
  rubyforge_project:
135
- rubygems_version: 1.8.23
123
+ rubygems_version: 2.2.2
136
124
  signing_key:
137
- specification_version: 3
125
+ specification_version: 4
138
126
  summary: AWS Auto-Scaling Assistant
139
127
  test_files: []
140
128
  has_rdoc: