scaltainer 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rspec +1 -0
- data/.travis.yml +7 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +133 -0
- data/Rakefile +8 -0
- data/bin/console +14 -0
- data/bin/setup +6 -0
- data/exe/scaltainer +12 -0
- data/lib/scaltainer/command.rb +31 -0
- data/lib/scaltainer/docker/service.rb +27 -0
- data/lib/scaltainer/exceptions.rb +6 -0
- data/lib/scaltainer/newrelic/metrics.rb +41 -0
- data/lib/scaltainer/runner.rb +111 -0
- data/lib/scaltainer/service_types/base.rb +72 -0
- data/lib/scaltainer/service_types/web.rb +49 -0
- data/lib/scaltainer/service_types/worker.rb +35 -0
- data/lib/scaltainer/service_types.rb +3 -0
- data/lib/scaltainer/version.rb +3 -0
- data/lib/scaltainer.rb +7 -0
- data/scaltainer.gemspec +33 -0
- metadata +167 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: d8a33c9f8383c9b06b2c6407f5700cb32f53541d
|
4
|
+
data.tar.gz: 5887f88f04df613831025a986154667afbe80091
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 7f7a731402f9d58926023a5de82b8f9c5f604c664a6e2cae85ff531fb0ebae71528ebd761adbab88d99bbae8b9887c936046192fe425e19e3fb0845294a42892
|
7
|
+
data.tar.gz: 036fea9e0e831ad8ef760f357726834eafd9ea2cb8bfe80c2e0e43468947cf1ec8d98872cf34066197dc9c3c2bbf25327da05a00221da367ea4e17cddf12e75b
|
data/.gitignore
ADDED
data/.rspec
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
--require spec_helper
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2017 Qatar Computing Research Institute, member of Qatar Foundation
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,133 @@
|
|
1
|
+
[](https://travis-ci.org/hammady/scaltainer)
|
2
|
+
[](https://coveralls.io/github/hammady/scaltainer?branch=master)
|
3
|
+
|
4
|
+
# Scaltainer
|
5
|
+
|
6
|
+
A Ruby gem to monitor docker swarm mode services and auto-scale them based on user configuration.
|
7
|
+
It can be used to monitor web services and worker services. The web services type has metrics like response time using [New Relic](https://newrelic.com/). The worker services type metrics are basically the queue size for each.
|
8
|
+
This gem is inspired by [HireFire](https://manager.hirefire.io/) and was indeed motivated by the migration
|
9
|
+
from [Heroku](https://www.heroku.com/) to Docker Swarm mode.
|
10
|
+
|
11
|
+
## Installation
|
12
|
+
|
13
|
+
Add this line to your application's Gemfile:
|
14
|
+
|
15
|
+
```ruby
|
16
|
+
gem 'scaltainer'
|
17
|
+
```
|
18
|
+
|
19
|
+
And then execute:
|
20
|
+
|
21
|
+
$ bundle
|
22
|
+
|
23
|
+
Or install it yourself as:
|
24
|
+
|
25
|
+
$ gem install scaltainer
|
26
|
+
|
27
|
+
## Usage
|
28
|
+
|
29
|
+
scaltainer
|
30
|
+
|
31
|
+
This will do a one-time check on the running service replicas and sends scaling out/in commands to the swarm cluster as appropriate.
|
32
|
+
Configuration is read from `scaltainer.yml` by default. If you want to read from another file add `-f yourconfig.yml`:
|
33
|
+
|
34
|
+
scaltainer -f yourconfig.yml
|
35
|
+
|
36
|
+
Note that after each run a new file is created (`yourconfig.yml.state`) which stores the state of the previous run.
|
37
|
+
This is because there are some configuration parameters (like sensitivity) need to
|
38
|
+
remember previous runs.
|
39
|
+
If you want to specify a different location for the state file, add the `--state-file` parameter.
|
40
|
+
Example:
|
41
|
+
|
42
|
+
scaltainer -f /path/to/configuration/file.yml --state-file /path/to/different/state/file.yml
|
43
|
+
|
44
|
+
Typically, the above command should be put inside a cronjob that is triggered every minute or so.
|
45
|
+
|
46
|
+
## Configuration
|
47
|
+
|
48
|
+
### Environment variables
|
49
|
+
|
50
|
+
- `DOCKER_URL`: Should point to the docker engine URL.
|
51
|
+
If not set, it defaults to local unix socket.
|
52
|
+
|
53
|
+
- `HIREFIRE_TOKEN`: If your application is configured the
|
54
|
+
[hirefire](https://help.hirefire.io/guides/hirefire/job-queue-any-programming-language) way, you need to
|
55
|
+
set `HIREFIRE_TOKEN` environment variable before invoking
|
56
|
+
`scaltainer`. This is used when probing your application
|
57
|
+
endpoint (see below) to get the number of jobs per queue
|
58
|
+
for each worker.
|
59
|
+
|
60
|
+
- `NEW_RELIC_LICENSE_KEY`: New Relic license key. Currently New Relic
|
61
|
+
is used to retrieve average response time metric for web services.
|
62
|
+
More monitoring services can be added in the future.
|
63
|
+
|
64
|
+
- `RESPONSE_TIME_WINDOW`: Time window in minutes to measure
|
65
|
+
average response time till the moment. For example 3 means
|
66
|
+
measure average response time in the past 3 minutes. Default value is 5.
|
67
|
+
|
68
|
+
- `LOG_LEVEL`: Accepted values here are: `DEBUG`, `INFO` (default), `WARN`, `ERROR`, `FATAL`.
|
69
|
+
Log output goes to stdout.
|
70
|
+
|
71
|
+
### Configuration file
|
72
|
+
|
73
|
+
The configuration file (determined by `-f FILE` command line parameter) should be in the following form:
|
74
|
+
|
75
|
+
# to get worker metrics
|
76
|
+
endpoint: https://your-app.com/hirefire/$HIREFIRE_TOKEN/info
|
77
|
+
# optional docker swarm stack name
|
78
|
+
stack_name: mystack
|
79
|
+
# list of web services to monitor
|
80
|
+
web_services:
|
81
|
+
# each service name should match docker service name
|
82
|
+
web:
|
83
|
+
# New Relic application id (required)
|
84
|
+
newrelic_app_id: <app_id>
|
85
|
+
# minimum replicas to maintain (default: 0)
|
86
|
+
min: 1
|
87
|
+
# maximum replicas to maintain (default: unlimited)
|
88
|
+
max: 5
|
89
|
+
# maximum response time above which to scale up (required)
|
90
|
+
max_response_time: 300
|
91
|
+
# minimum response time below which to scale down (required)
|
92
|
+
min_response_time: 100
|
93
|
+
# replica quantitiy to scale up at a time (default: 1)
|
94
|
+
upscale_quantity: 2
|
95
|
+
# replica quantitiy to scale down at a time (default: 1)
|
96
|
+
downscale_quantity: 1
|
97
|
+
# number of breaches to wait for before scaling up (default: 1)
|
98
|
+
upscale_sensitivity: 1
|
99
|
+
# number of breaches to wait for before scaling down (default: 1)
|
100
|
+
downscale_sensitivity: 1
|
101
|
+
webapi:
|
102
|
+
...
|
103
|
+
worker_services:
|
104
|
+
worker1:
|
105
|
+
min: 1
|
106
|
+
max: 10
|
107
|
+
# number of jobs each worker replica should process (required)
|
108
|
+
# the bigger the ratio, the less number of workers scaled out
|
109
|
+
ratio: 3
|
110
|
+
upscale_sensitivity: 1
|
111
|
+
downscale_sensitivity: 1
|
112
|
+
worker2:
|
113
|
+
...
|
114
|
+
|
115
|
+
More details about configuration parameters can be found in [HireFire docs](https://help.hirefire.io/guides).
|
116
|
+
|
117
|
+
## Development
|
118
|
+
|
119
|
+
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
120
|
+
|
121
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
122
|
+
|
123
|
+
## Contributing
|
124
|
+
|
125
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/hammady/scaltainer.
|
126
|
+
|
127
|
+
## TODOs
|
128
|
+
|
129
|
+
- Rspec
|
130
|
+
|
131
|
+
## License
|
132
|
+
|
133
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "scaltainer"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start(__FILE__)
|
data/bin/setup
ADDED
data/exe/scaltainer
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scaltainer'
|
4
|
+
|
5
|
+
begin
|
6
|
+
configfile, statefile, logger = Scaltainer::Command.parse ARGV
|
7
|
+
Scaltainer::Runner.new configfile, statefile, logger
|
8
|
+
rescue => e
|
9
|
+
$stderr.puts e.message
|
10
|
+
$stderr.puts e.backtrace
|
11
|
+
exit 1
|
12
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require "logger"
|
2
|
+
require "optparse"
|
3
|
+
|
4
|
+
module Scaltainer
|
5
|
+
class Command
|
6
|
+
def self.parse(args)
|
7
|
+
configfile, statefile = 'scaltainer.yml', nil
|
8
|
+
OptionParser.new do |opts|
|
9
|
+
opts.banner = "Usage: scaltainer [options]"
|
10
|
+
opts.on("-f", "--conf-file FILE", "Specify configuration file (default: scaltainer.yml)") do |file|
|
11
|
+
configfile = file
|
12
|
+
end
|
13
|
+
opts.on("--state-file FILE", "Specify state file (default: <conf-file>.state)") do |file|
|
14
|
+
statefile = file
|
15
|
+
end
|
16
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
17
|
+
puts opts
|
18
|
+
exit
|
19
|
+
end
|
20
|
+
end.parse!
|
21
|
+
|
22
|
+
statefile = "#{configfile}.state" unless statefile
|
23
|
+
|
24
|
+
raise ConfigurationError.new("File not found: #{configfile}") unless File.exists?(configfile)
|
25
|
+
logger = Logger.new(STDOUT)
|
26
|
+
logger.level = %w(debug info warn error fatal unknown).find_index((ENV['LOG_LEVEL'] || '').downcase) || 1
|
27
|
+
|
28
|
+
return configfile, statefile, logger
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# source: https://github.com/Stazer/docker-api/blob/feature/swarm_support/lib/docker/service.rb
|
2
|
+
|
3
|
+
# This class represents a Docker Service. It's important to note that nothing
|
4
|
+
# is cached so that the information is always up to date.
|
5
|
+
require "docker"
|
6
|
+
|
7
|
+
class Docker::Service
|
8
|
+
include Docker::Base
|
9
|
+
|
10
|
+
def self.all(opts = {}, conn = Docker.connection)
|
11
|
+
hashes = Docker::Util.parse_json(conn.get('/services', opts)) || []
|
12
|
+
hashes.map { |hash| new(conn, hash) }
|
13
|
+
end
|
14
|
+
|
15
|
+
def update(opts)
|
16
|
+
version = self.info["Version"]["Index"]
|
17
|
+
connection.post("/services/#{self.id}/update", {version: version}, body: opts.to_json)
|
18
|
+
end
|
19
|
+
|
20
|
+
def scale(replicas)
|
21
|
+
spec = self.info["Spec"]
|
22
|
+
spec["Mode"]["Replicated"]["Replicas"] = replicas
|
23
|
+
update(spec)
|
24
|
+
end
|
25
|
+
|
26
|
+
private_class_method :new
|
27
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
module Newrelic
|
2
|
+
class Metrics
|
3
|
+
def initialize(license_key)
|
4
|
+
@headers = {"X-Api-Key" => license_key}
|
5
|
+
@base_url = "https://api.newrelic.com/v2"
|
6
|
+
end
|
7
|
+
|
8
|
+
# https://docs.newrelic.com/docs/apis/rest-api-v2/application-examples-v2/average-response-time-examples-v2
|
9
|
+
def get_avg_response_time(app_id, from, to)
|
10
|
+
url = "#{@base_url}/applications/#{app_id}/metrics/data.json"
|
11
|
+
conn = Excon.new(url, persistent: true, tcp_nodelay: true)
|
12
|
+
time_range = "from=#{from.iso8601}&to=#{to.iso8601}"
|
13
|
+
metric_names_array = %w(
|
14
|
+
names[]=HttpDispatcher&values[]=average_call_time&values[]=call_count
|
15
|
+
names[]=WebFrontend/QueueTime&values[]=call_count&values[]=average_response_time
|
16
|
+
)
|
17
|
+
response_array = request(conn, metric_names_array, time_range)
|
18
|
+
http_call_count, http_average_call_time = response_array[0]["call_count"], response_array[0]["average_call_time"]
|
19
|
+
webfe_call_count, webfe_average_response_time = response_array[1]["call_count"], response_array[1]["average_response_time"]
|
20
|
+
|
21
|
+
http_average_call_time + (1.0 * webfe_call_count * webfe_average_response_time / http_call_count)
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def request(conn, metric_names_array, time_range)
|
27
|
+
requests = metric_names_array.map {|metric_names|
|
28
|
+
{
|
29
|
+
method: :get, headers: @headers,
|
30
|
+
query: "#{metric_names}&#{time_range}&summarize=true"
|
31
|
+
}
|
32
|
+
}
|
33
|
+
responses = conn.requests requests
|
34
|
+
responses.map {|response|
|
35
|
+
body = JSON.parse(response.body)
|
36
|
+
body["metric_data"]["metrics"][0]["timeslices"][0]["values"]
|
37
|
+
}
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
41
|
+
end
|
@@ -0,0 +1,111 @@
|
|
1
|
+
require "yaml"
|
2
|
+
|
3
|
+
module Scaltainer
|
4
|
+
class Runner
|
5
|
+
def initialize(configfile, statefile, logger)
|
6
|
+
@logger = logger
|
7
|
+
@default_service_config = {
|
8
|
+
"min" => 0,
|
9
|
+
"upscale_quantity" => 1,
|
10
|
+
"downscale_quantity" => 1,
|
11
|
+
"upscale_sensitivity" => 1,
|
12
|
+
"downscale_sensitivity" => 1
|
13
|
+
}
|
14
|
+
@logger.debug "Scaltainer initialized with configuration file: #{configfile}, and state file: #{statefile}"
|
15
|
+
config = YAML.load_file configfile
|
16
|
+
Docker.logger = @logger
|
17
|
+
state = get_state(statefile) || {}
|
18
|
+
endpoint = config["endpoint"]
|
19
|
+
service_prefix = config["stack_name"]
|
20
|
+
iterate_services config["web_services"], service_prefix, ServiceTypeWeb.new(endpoint), state
|
21
|
+
iterate_services config["worker_services"], service_prefix, ServiceTypeWorker.new(endpoint), state
|
22
|
+
save_state statefile, state
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
def get_state(statefile)
|
28
|
+
YAML.load_file statefile if File.exists? statefile
|
29
|
+
end
|
30
|
+
|
31
|
+
def save_state(statefile, state)
|
32
|
+
File.write(statefile, state.to_yaml)
|
33
|
+
end
|
34
|
+
|
35
|
+
def get_service(service_name)
|
36
|
+
begin
|
37
|
+
service = Docker::Service.all(filters: {name: [service_name]}.to_json)[0]
|
38
|
+
rescue => e
|
39
|
+
raise NetworkError.new "Could not get service with name #{service_name} from docker engine at #{Docker.url}.\n#{e.message}"
|
40
|
+
end
|
41
|
+
raise ConfigurationError.new "Unknown service to docker: #{service_name}" unless service
|
42
|
+
service
|
43
|
+
end
|
44
|
+
|
45
|
+
def get_service_replicas(service)
|
46
|
+
# ask docker about replicas for service
|
47
|
+
replicated = service.info["Spec"]["Mode"]["Replicated"]
|
48
|
+
raise ConfigurationError.new "Cannot replicate a global service: #{service.info['Spec']['Name']}" unless replicated
|
49
|
+
replicated["Replicas"]
|
50
|
+
end
|
51
|
+
|
52
|
+
def iterate_services(services, service_prefix, type, state)
|
53
|
+
begin
|
54
|
+
metrics = type.get_metrics services
|
55
|
+
@logger.debug "Retrieved metrics for #{type} services: #{metrics}"
|
56
|
+
services.each do |service_name, service_config|
|
57
|
+
begin
|
58
|
+
state[service_name] ||= {}
|
59
|
+
service_state = state[service_name]
|
60
|
+
@logger.debug "Service #{service_name} currently has state: #{service_state}"
|
61
|
+
service_config = @default_service_config.merge service_config
|
62
|
+
@logger.debug "Service #{service_name} configuration: #{service_config}"
|
63
|
+
process_service service_name, service_config, service_state, service_prefix, type, metrics
|
64
|
+
rescue RuntimeError => e
|
65
|
+
# skipping service
|
66
|
+
log_exception e
|
67
|
+
end
|
68
|
+
end
|
69
|
+
rescue RuntimeError => e
|
70
|
+
# skipping service type
|
71
|
+
log_exception e
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def log_exception(e)
|
76
|
+
@logger.log (e.class == Scaltainer::Warning ? Logger::WARN : Logger::ERROR), e.message
|
77
|
+
end
|
78
|
+
|
79
|
+
def process_service(service_name, config, state, prefix, type, metrics)
|
80
|
+
full_service_name = prefix ? "#{prefix}_#{service_name}" : service_name
|
81
|
+
service = get_service full_service_name
|
82
|
+
@logger.debug "Found service at docker with name '#{service_name}' and id '#{service.id}'"
|
83
|
+
current_replicas = get_service_replicas service
|
84
|
+
@logger.debug "Service #{service_name} is currently configured for #{current_replicas} replica(s)"
|
85
|
+
metric = metrics[service_name]
|
86
|
+
raise Scaltainer::Warning.new("Configured service '#{service_name}' not found in metrics endpoint") unless metric
|
87
|
+
desired_replicas = type.determine_desired_replicas metric, config, current_replicas
|
88
|
+
@logger.debug "Desired number of replicas for service #{service_name} is #{desired_replicas}"
|
89
|
+
adjusted_replicas = type.adjust_desired_replicas(desired_replicas, config)
|
90
|
+
@logger.debug "Desired number of replicas for service #{service_name} is adjusted to #{adjusted_replicas}"
|
91
|
+
replica_diff = desired_replicas - current_replicas
|
92
|
+
type.yield_to_scale(replica_diff, config, state, metric,
|
93
|
+
service_name, @logger) do
|
94
|
+
scale_out service, current_replicas, adjusted_replicas
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def scale_out(service, current_replicas, desired_replicas)
|
99
|
+
return if current_replicas == desired_replicas
|
100
|
+
# send scale command to docker
|
101
|
+
service_name = service.info['Spec']['Name']
|
102
|
+
@logger.info "Scaling #{service_name} from #{current_replicas} to #{desired_replicas}"
|
103
|
+
begin
|
104
|
+
service.scale desired_replicas
|
105
|
+
rescue => e
|
106
|
+
raise NetworkError.new "Could not scale service #{service_name} due to docker engine error at #{Docker.url}.\n#{e.message}"
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
end # class
|
111
|
+
end # module
|
@@ -0,0 +1,72 @@
|
|
1
|
+
module Scaltainer
|
2
|
+
class ServiceTypeBase
|
3
|
+
def initialize(app_endpoint)
|
4
|
+
@app_endpoint = app_endpoint.sub('$HIREFIRE_TOKEN', ENV['HIREFIRE_TOKEN'] || '') if app_endpoint
|
5
|
+
end
|
6
|
+
|
7
|
+
def get_metrics(services)
|
8
|
+
services_count = services.keys.length rescue 0
|
9
|
+
raise Scaltainer::Warning.new "No services found for #{self.class.name}" if services_count == 0
|
10
|
+
end
|
11
|
+
|
12
|
+
def determine_desired_replicas(metric, service_config, current_replicas)
|
13
|
+
raise ConfigurationError.new 'No metric found for requested service' unless metric
|
14
|
+
raise ConfigurationError.new 'No configuration found for requested service' unless service_config
|
15
|
+
end
|
16
|
+
|
17
|
+
def adjust_desired_replicas(desired_replicas, config)
|
18
|
+
desired_replicas = [config["max"], desired_replicas].min if config["max"]
|
19
|
+
[config["min"], desired_replicas].max
|
20
|
+
end
|
21
|
+
|
22
|
+
def yield_to_scale(replica_diff, config, state, metric, service_name, logger)
|
23
|
+
# Force up/down when below/above min/max?
|
24
|
+
# one could argue that this could only happen on first deployment or when manually
|
25
|
+
# scaled outside the scope of scaltainer. It is OK in this case to still apply sensitivity rules
|
26
|
+
if replica_diff > 0
|
27
|
+
# breach: change state and scale up
|
28
|
+
state["upscale_sensitivity"] ||= 0
|
29
|
+
state["upscale_sensitivity"] += 1
|
30
|
+
state["downscale_sensitivity"] = 0
|
31
|
+
if state["upscale_sensitivity"] >= config["upscale_sensitivity"]
|
32
|
+
yield
|
33
|
+
state["upscale_sensitivity"] = 0
|
34
|
+
else
|
35
|
+
logger.debug "Scaling up of service #{service_name} blocked by upscale_sensitivity at level " +
|
36
|
+
"#{state["upscale_sensitivity"]} while level #{config["upscale_sensitivity"]} is required"
|
37
|
+
end
|
38
|
+
elsif replica_diff < 0 # TODO force down when above max?
|
39
|
+
# breach: change state and scale down
|
40
|
+
if can_scale_down? metric, config
|
41
|
+
state["downscale_sensitivity"] ||= 0
|
42
|
+
state["downscale_sensitivity"] += 1
|
43
|
+
state["upscale_sensitivity"] = 0
|
44
|
+
if state["downscale_sensitivity"] >= config["downscale_sensitivity"]
|
45
|
+
yield
|
46
|
+
state["downscale_sensitivity"] = 0
|
47
|
+
else
|
48
|
+
logger.debug "Scaling down of service #{service_name} blocked by downscale_sensitivity at level " +
|
49
|
+
"#{state["downscale_sensitivity"]} while level #{config["downscale_sensitivity"]} is required"
|
50
|
+
end
|
51
|
+
else
|
52
|
+
logger.debug "Scaling down of service #{service_name} to #{metric} replicas blocked by a non-decrementable config"
|
53
|
+
end
|
54
|
+
else
|
55
|
+
# no breach, change state
|
56
|
+
state["upscale_sensitivity"] = 0
|
57
|
+
state["downscale_sensitivity"] = 0
|
58
|
+
logger.info "No need to scale service #{service_name}"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def to_s
|
63
|
+
"Base"
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
def can_scale_down?(metric, config)
|
69
|
+
self.class == ServiceTypeWeb || metric == 0 || config["decrementable"]
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module Scaltainer
|
2
|
+
class ServiceTypeWeb < ServiceTypeBase
|
3
|
+
def initialize(app_endpoint = nil)
|
4
|
+
super
|
5
|
+
end
|
6
|
+
|
7
|
+
def get_metrics(services)
|
8
|
+
super
|
9
|
+
nr_key = ENV['NEW_RELIC_LICENSE_KEY']
|
10
|
+
raise ConfigurationError.new 'NEW_RELIC_LICENSE_KEY not set in environment' unless nr_key
|
11
|
+
nr = Newrelic::Metrics.new nr_key
|
12
|
+
to = Time.now
|
13
|
+
from = to - (ENV['RESPONSE_TIME_WINDOW'] || '5').to_i * 60
|
14
|
+
|
15
|
+
services.reduce({}) do |hash, (service_name, service_config)|
|
16
|
+
app_id = service_config["newrelic_app_id"]
|
17
|
+
raise ConfigurationError.new "Service #{service_name} does not have a corresponding newrelic_app_id" unless app_id
|
18
|
+
|
19
|
+
begin
|
20
|
+
metric = nr.get_avg_response_time app_id, from, to
|
21
|
+
rescue => e
|
22
|
+
raise NetworkError.new "Could not retrieve metrics from New Relic API for #{service_name}.\n#{e.message}"
|
23
|
+
end
|
24
|
+
|
25
|
+
hash.merge!(service_name => metric)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def determine_desired_replicas(metric, service_config, current_replicas)
|
30
|
+
super
|
31
|
+
raise ConfigurationError.new "Missing max_response_time in web service configuration" unless service_config["max_response_time"]
|
32
|
+
raise ConfigurationError.new "Missing min_response_time in web service configuration" unless service_config["min_response_time"]
|
33
|
+
unless service_config["min_response_time"] <= service_config["max_response_time"]
|
34
|
+
raise ConfigurationError.new "min_response_time and max_response_time are not in order"
|
35
|
+
end
|
36
|
+
desired_replicas = if metric > service_config["max_response_time"]
|
37
|
+
current_replicas + service_config["upscale_quantity"]
|
38
|
+
elsif metric < service_config["min_response_time"]
|
39
|
+
current_replicas - service_config["downscale_quantity"]
|
40
|
+
else
|
41
|
+
current_replicas
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def to_s
|
46
|
+
"Web"
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Scaltainer
|
2
|
+
class ServiceTypeWorker < ServiceTypeBase
|
3
|
+
def initialize(app_endpoint = nil)
|
4
|
+
super
|
5
|
+
end
|
6
|
+
|
7
|
+
def get_metrics(services)
|
8
|
+
super
|
9
|
+
begin
|
10
|
+
response = Excon.get(@app_endpoint)
|
11
|
+
m = JSON.parse(response.body)
|
12
|
+
m.reduce({}){|hash, item| hash.merge!({item["name"] => item["quantity"]})}
|
13
|
+
rescue JSON::ParserError => e
|
14
|
+
raise ConfigurationError.new "app_endpoint returned non json response: #{response.body[0..128]}"
|
15
|
+
rescue TypeError => e
|
16
|
+
raise ConfigurationError.new "app_endpoint returned unexpected json response: #{response.body[0..128]}"
|
17
|
+
rescue => e
|
18
|
+
raise NetworkError.new "Could not retrieve metrics from application endpoint: #{@app_endpoint}.\n#{e.message}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def determine_desired_replicas(metric, service_config, current_replicas)
|
23
|
+
super
|
24
|
+
raise ConfigurationError.new "Missing ratio in worker service configuration" unless service_config["ratio"]
|
25
|
+
if !metric.is_a?(Integer) || metric < 0
|
26
|
+
raise ConfigurationError.new "#{metric} is an invalid metric value, must be a non-negative number"
|
27
|
+
end
|
28
|
+
desired_replicas = (metric * 1.0 / service_config["ratio"]).ceil
|
29
|
+
end
|
30
|
+
|
31
|
+
def to_s
|
32
|
+
"Worker"
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
data/lib/scaltainer.rb
ADDED
data/scaltainer.gemspec
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require "scaltainer/version"
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "scaltainer"
|
8
|
+
spec.version = Scaltainer::VERSION
|
9
|
+
spec.authors = ["Hossam Hammady"]
|
10
|
+
spec.email = ["github@hammady.net"]
|
11
|
+
|
12
|
+
spec.summary = %q{Autoscale docker swarm services based on application metrics and more}
|
13
|
+
spec.description = %q{A ruby gem inspired by HireFire to autoscale docker swarm services.
|
14
|
+
Metrics can be standard average response time, New Relic web metrics, queue size for workers, ...}
|
15
|
+
spec.homepage = "https://github.com/hammady/scaltainer"
|
16
|
+
spec.license = "MIT"
|
17
|
+
|
18
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
19
|
+
f.match(%r{^(test|spec|features)/})
|
20
|
+
end
|
21
|
+
spec.bindir = "exe"
|
22
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
23
|
+
spec.require_paths = ["lib"]
|
24
|
+
|
25
|
+
spec.add_development_dependency "bundler", "~> 1.15"
|
26
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
27
|
+
spec.add_development_dependency 'rspec', '~> 3.5'
|
28
|
+
spec.add_development_dependency 'coderay', '~> 1.1'
|
29
|
+
spec.add_development_dependency 'coveralls', '~> 0.8'
|
30
|
+
|
31
|
+
spec.add_runtime_dependency 'excon', '>= 0.47.0'
|
32
|
+
spec.add_runtime_dependency "docker-api"
|
33
|
+
end
|
metadata
ADDED
@@ -0,0 +1,167 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: scaltainer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.0
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Hossam Hammady
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2017-11-02 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.15'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.15'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.5'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.5'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: coderay
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.1'
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.1'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: coveralls
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '0.8'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0.8'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: excon
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: 0.47.0
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: 0.47.0
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: docker-api
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :runtime
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
111
|
+
description: |-
|
112
|
+
A ruby gem inspired by HireFire to autoscale docker swarm services.
|
113
|
+
Metrics can be standard average response time, New Relic web metrics, queue size for workers, ...
|
114
|
+
email:
|
115
|
+
- github@hammady.net
|
116
|
+
executables:
|
117
|
+
- scaltainer
|
118
|
+
extensions: []
|
119
|
+
extra_rdoc_files: []
|
120
|
+
files:
|
121
|
+
- ".gitignore"
|
122
|
+
- ".rspec"
|
123
|
+
- ".travis.yml"
|
124
|
+
- Gemfile
|
125
|
+
- LICENSE.txt
|
126
|
+
- README.md
|
127
|
+
- Rakefile
|
128
|
+
- bin/console
|
129
|
+
- bin/setup
|
130
|
+
- exe/scaltainer
|
131
|
+
- lib/scaltainer.rb
|
132
|
+
- lib/scaltainer/command.rb
|
133
|
+
- lib/scaltainer/docker/service.rb
|
134
|
+
- lib/scaltainer/exceptions.rb
|
135
|
+
- lib/scaltainer/newrelic/metrics.rb
|
136
|
+
- lib/scaltainer/runner.rb
|
137
|
+
- lib/scaltainer/service_types.rb
|
138
|
+
- lib/scaltainer/service_types/base.rb
|
139
|
+
- lib/scaltainer/service_types/web.rb
|
140
|
+
- lib/scaltainer/service_types/worker.rb
|
141
|
+
- lib/scaltainer/version.rb
|
142
|
+
- scaltainer.gemspec
|
143
|
+
homepage: https://github.com/hammady/scaltainer
|
144
|
+
licenses:
|
145
|
+
- MIT
|
146
|
+
metadata: {}
|
147
|
+
post_install_message:
|
148
|
+
rdoc_options: []
|
149
|
+
require_paths:
|
150
|
+
- lib
|
151
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
152
|
+
requirements:
|
153
|
+
- - ">="
|
154
|
+
- !ruby/object:Gem::Version
|
155
|
+
version: '0'
|
156
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
157
|
+
requirements:
|
158
|
+
- - ">="
|
159
|
+
- !ruby/object:Gem::Version
|
160
|
+
version: '0'
|
161
|
+
requirements: []
|
162
|
+
rubyforge_project:
|
163
|
+
rubygems_version: 2.5.1
|
164
|
+
signing_key:
|
165
|
+
specification_version: 4
|
166
|
+
summary: Autoscale docker swarm services based on application metrics and more
|
167
|
+
test_files: []
|