scaltainer 0.1.5 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.dockerignore +3 -0
- data/.gitignore +5 -1
- data/Dockerfile +4 -2
- data/README.md +124 -32
- data/exe/scaltainer +2 -2
- data/lib/scaltainer.rb +1 -1
- data/lib/scaltainer/command.rb +19 -2
- data/lib/scaltainer/orchestrators.rb +3 -0
- data/lib/scaltainer/orchestrators/base.rb +17 -0
- data/lib/scaltainer/orchestrators/kubernetes.rb +84 -0
- data/lib/scaltainer/{docker/service.rb → orchestrators/swarm.rb} +23 -0
- data/lib/scaltainer/runner.rb +39 -43
- data/lib/scaltainer/service_types/base.rb +7 -7
- data/lib/scaltainer/service_types/web.rb +3 -3
- data/lib/scaltainer/service_types/worker.rb +1 -1
- data/lib/scaltainer/version.rb +1 -1
- data/scaltainer.gemspec +1 -0
- metadata +21 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 642f49828031f354f271d0ad4e28268ab40e1f1a
|
4
|
+
data.tar.gz: 2be74e2694d99bc5edde992e8e277867db0de44b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 34a7ddda22e20d4e909a29c7b2183344712a7fc72dd8d970055c578834e6d1707496bfeebfd09cc4224d62a806615756cb964bddb07653b60798ded0ffe0cbe8
|
7
|
+
data.tar.gz: 07b3b919496e8cc1fbdbc25199e53509e5bb44b3be47a0257430ce6be22102dc08eb43e7d6a2e9bc2eb6ef2ad8d0d33b84ebec9a5c4be8ee83b5094df7e25bc5
|
data/.dockerignore
ADDED
data/.gitignore
CHANGED
data/Dockerfile
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
FROM ruby:2.3
|
2
2
|
|
3
|
-
|
3
|
+
label maintainer="Hossam Hammady <github@hammady.net>"
|
4
4
|
|
5
|
-
|
5
|
+
WORKDIR /home
|
6
|
+
COPY / /home/
|
7
|
+
RUN bundle install && bundle exec rake install
|
6
8
|
|
7
9
|
ENTRYPOINT ["scaltainer"]
|
8
10
|
|
data/README.md
CHANGED
@@ -4,32 +4,38 @@
|
|
4
4
|
|
5
5
|
# Scaltainer
|
6
6
|
|
7
|
-
A Ruby gem to monitor
|
7
|
+
A Ruby gem to monitor Docker Swarm mode services and Kubernetes resources
|
8
|
+
and auto-scale them based on user configuration.
|
8
9
|
It can be used to monitor web services and worker services. The web services type has metrics like response time using [New Relic](https://newrelic.com/). The worker services type metrics are basically the queue size for each.
|
9
10
|
This gem is inspired by [HireFire](https://manager.hirefire.io/) and was indeed motivated by the migration
|
10
|
-
from [Heroku](https://www.heroku.com/) to Docker
|
11
|
+
from [Heroku](https://www.heroku.com/) to Docker.
|
11
12
|
|
12
13
|
## Installation
|
13
14
|
|
14
15
|
Add this line to your application's Gemfile:
|
15
16
|
|
16
|
-
|
17
|
-
gem 'scaltainer'
|
18
|
-
```
|
17
|
+
Install using rubygems:
|
19
18
|
|
20
|
-
|
19
|
+
$ gem install scaltainer
|
21
20
|
|
22
|
-
|
21
|
+
## Usage
|
23
22
|
|
24
|
-
|
23
|
+
For Docker swarm:
|
25
24
|
|
26
|
-
|
25
|
+
scaltainer -o swarm
|
27
26
|
|
28
|
-
|
27
|
+
Or simply:
|
29
28
|
|
30
29
|
scaltainer
|
31
30
|
|
32
|
-
|
31
|
+
For Kubernetes:
|
32
|
+
|
33
|
+
scaltainer -o kubernetes
|
34
|
+
|
35
|
+
|
36
|
+
This will do a one-time check on the running docker service replicas
|
37
|
+
or Kubernetes replication controllers, replica sets, or deployments.
|
38
|
+
Then it sends scaling out/in commands to the cluster as appropriate.
|
33
39
|
Configuration is read from `scaltainer.yml` by default. If you want to read from another file add `-f yourconfig.yml`:
|
34
40
|
|
35
41
|
scaltainer -f yourconfig.yml
|
@@ -47,15 +53,44 @@ specify the wait time between repetitions using the `-w` parameter in seconds:
|
|
47
53
|
|
48
54
|
scaltainer -w 60
|
49
55
|
|
50
|
-
This will repeatedly call scaltainer every 60 seconds, sleeping in
|
56
|
+
This will repeatedly call scaltainer every 60 seconds, sleeping in-between.
|
51
57
|
|
52
58
|
## Configuration
|
53
59
|
|
54
60
|
### Environment variables
|
55
61
|
|
62
|
+
#### Docker swarm options
|
63
|
+
|
56
64
|
- `DOCKER_URL`: Should point to the docker engine URL.
|
57
65
|
If not set, it defaults to local unix socket.
|
58
66
|
|
67
|
+
#### Kubernetes options
|
68
|
+
|
69
|
+
- `KUBECONFIG`: set to Kubernetes config
|
70
|
+
(default: `$HOME/.kube/config`) if you want to connect
|
71
|
+
to the current configured cluster.
|
72
|
+
|
73
|
+
- `KUBERNETES_API_SERVER`: overrides option in `KUBECONFIG`
|
74
|
+
and defaults to `https://kubernetes.default:443`.
|
75
|
+
|
76
|
+
- `KUBERNETES_SKIP_SSL_VERIFY`: `KUBECONFIG` option overrides
|
77
|
+
this, set to any value to skip SSL verification.
|
78
|
+
|
79
|
+
- `KUBERNETES_API_ENDPOINT`: defaults to `/api`.
|
80
|
+
|
81
|
+
- `KUBERNETES_API_VERSION`: overrides option in `KUBECONFIG`
|
82
|
+
and defaults to `v1`.
|
83
|
+
|
84
|
+
- `KUBERNETES_CONTROLLER_KIND`: controller kind to scale,
|
85
|
+
allowed values: `deployment` (default),
|
86
|
+
`replication_controller`, or `replica_set`.
|
87
|
+
|
88
|
+
Make sure the `KUBERNETES_CONTROLLER_KIND` you specify is
|
89
|
+
part of the api specified using `KUBERNETES_API_ENDPOINT`
|
90
|
+
and `KUBERNETES_API_VERSION`.
|
91
|
+
|
92
|
+
#### General options
|
93
|
+
|
59
94
|
- `HIREFIRE_TOKEN`: If your application is configured the
|
60
95
|
[hirefire](https://help.hirefire.io/guides/hirefire/job-queue-any-programming-language) way, you need to
|
61
96
|
set `HIREFIRE_TOKEN` environment variable before invoking
|
@@ -86,8 +121,8 @@ The configuration file (determined by `-f FILE` command line parameter) should b
|
|
86
121
|
|
87
122
|
# to get worker metrics
|
88
123
|
endpoint: https://your-app.com/hirefire/$HIREFIRE_TOKEN/info
|
89
|
-
# optional docker swarm stack name
|
90
|
-
|
124
|
+
# optional docker swarm stack name or kubernetes namespace
|
125
|
+
namespace: mynamespace
|
91
126
|
# list of web services to monitor
|
92
127
|
web_services:
|
93
128
|
# each service name should match docker service name
|
@@ -126,25 +161,15 @@ The configuration file (determined by `-f FILE` command line parameter) should b
|
|
126
161
|
|
127
162
|
More details about configuration parameters can be found in [HireFire docs](https://help.hirefire.io/guides).
|
128
163
|
|
129
|
-
## Docker
|
164
|
+
## Docker Swarm usage
|
130
165
|
|
131
|
-
|
132
|
-
|
133
|
-
docker run -it --rm rayyanqcri/scaltainer
|
134
|
-
|
135
|
-
Which will print the usage. To add arguments, just append them:
|
136
|
-
|
137
|
-
docker run -it --rm rayyanqcri/scaltainer -f scaltainer.yml
|
138
|
-
|
139
|
-
Scaltainer should typically be run as a minutely cron service.
|
140
|
-
If you are using [rayyanqcri/swarm-scheduler](https://github.com/rayyanqcri/swarm-scheduler),
|
141
|
-
a service definition for scaltainer is typically something like this:
|
166
|
+
A service definition for scaltainer is typically something like this:
|
142
167
|
|
143
168
|
version: '3.3'
|
144
169
|
services:
|
145
170
|
scaltainer:
|
146
171
|
image: rayyanqcri/scaltainer:latest
|
147
|
-
command: -f /scaltainer.yml --state-file /tmp/scaltainer-state.yml
|
172
|
+
command: -f /scaltainer.yml --state-file /tmp/scaltainer-state.yml -w 60
|
148
173
|
volumes:
|
149
174
|
- /var/run/docker.sock:/var/run/docker.sock
|
150
175
|
environment:
|
@@ -157,9 +182,7 @@ a service definition for scaltainer is typically something like this:
|
|
157
182
|
secrets:
|
158
183
|
- scaltainer
|
159
184
|
deploy:
|
160
|
-
replicas:
|
161
|
-
restart_policy:
|
162
|
-
condition: none
|
185
|
+
replicas: 1
|
163
186
|
placement:
|
164
187
|
constraints:
|
165
188
|
- node.role == manager
|
@@ -177,6 +200,75 @@ Where `scaltainer.env` is a file containing HireFire and NewRelic secrets:
|
|
177
200
|
|
178
201
|
And `scaltainer.yml` is the scaltainer configuration file.
|
179
202
|
|
203
|
+
## Kubernetes usage
|
204
|
+
|
205
|
+
### Create a ConfigMap
|
206
|
+
|
207
|
+
kubectl create configmap scaltainer --from-file=scaltainer.yaml=/path/to/your/scaltainer.yml
|
208
|
+
|
209
|
+
Where `/path/to/your/scaltainer.yml` is the scaltainer configuration file.
|
210
|
+
|
211
|
+
### Create a Secret
|
212
|
+
|
213
|
+
kubectl create secret generic scaltainer --from-env-file=/path/to/scaltainer.env
|
214
|
+
|
215
|
+
Where `/path/to/scaltainer.env` is a file containing HireFire and NewRelic secrets:
|
216
|
+
|
217
|
+
HIREFIRE_TOKEN=
|
218
|
+
NEW_RELIC_API_KEY=
|
219
|
+
|
220
|
+
### Create a Deployment:
|
221
|
+
|
222
|
+
kubectl apply -f scaltainer-kube.yaml
|
223
|
+
|
224
|
+
Where scaltainer-kube.yaml has the following content:
|
225
|
+
|
226
|
+
apiVersion: extensions/v1beta1
|
227
|
+
kind: Deployment
|
228
|
+
metadata:
|
229
|
+
labels:
|
230
|
+
app: scaltainer
|
231
|
+
name: scaltainer
|
232
|
+
spec:
|
233
|
+
replicas: 1
|
234
|
+
template:
|
235
|
+
metadata:
|
236
|
+
labels:
|
237
|
+
app: scaltainer
|
238
|
+
spec:
|
239
|
+
containers:
|
240
|
+
- image: rayyanqcri/scaltainer:latest
|
241
|
+
name: scaltainer
|
242
|
+
args:
|
243
|
+
- -o
|
244
|
+
- kubernetes
|
245
|
+
- -f
|
246
|
+
- /etc/config/scaltainer.yaml
|
247
|
+
- --state-file
|
248
|
+
- /tmp/scaltainer-state.yaml
|
249
|
+
- -w
|
250
|
+
- "60"
|
251
|
+
env:
|
252
|
+
- name: KUBERNETES_SKIP_SSL_VERIFY
|
253
|
+
value: "yes"
|
254
|
+
- name: KUBERNETES_API_ENDPOINT
|
255
|
+
value: /apis/extensions
|
256
|
+
- name: KUBERNETES_API_VERSION
|
257
|
+
value: v1beta1
|
258
|
+
- name: KUBERNETES_CONTROLLER_KIND
|
259
|
+
value: deployment
|
260
|
+
envFrom:
|
261
|
+
- secretRef:
|
262
|
+
name: scaltainer
|
263
|
+
volumeMounts:
|
264
|
+
- name: scaltainer-config
|
265
|
+
mountPath: "/etc/config"
|
266
|
+
volumes:
|
267
|
+
- name: scaltainer-config
|
268
|
+
configMap:
|
269
|
+
name: scaltainer
|
270
|
+
|
271
|
+
|
180
272
|
## Development
|
181
273
|
|
182
274
|
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
@@ -187,9 +279,9 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
187
279
|
|
188
280
|
Bug reports and pull requests are welcome on GitHub at https://github.com/hammady/scaltainer.
|
189
281
|
|
190
|
-
##
|
282
|
+
## Testing
|
191
283
|
|
192
|
-
|
284
|
+
rake
|
193
285
|
|
194
286
|
## License
|
195
287
|
|
data/exe/scaltainer
CHANGED
@@ -3,8 +3,8 @@
|
|
3
3
|
require 'scaltainer'
|
4
4
|
|
5
5
|
begin
|
6
|
-
configfile, statefile, logger, wait = Scaltainer::Command.parse ARGV
|
7
|
-
Scaltainer::Runner.new configfile, statefile, logger, wait
|
6
|
+
configfile, statefile, logger, wait, orchestrator = Scaltainer::Command.parse ARGV
|
7
|
+
Scaltainer::Runner.new configfile, statefile, logger, wait, orchestrator
|
8
8
|
rescue => e
|
9
9
|
$stderr.puts e.message
|
10
10
|
$stderr.puts e.backtrace
|
data/lib/scaltainer.rb
CHANGED
data/lib/scaltainer/command.rb
CHANGED
@@ -4,7 +4,7 @@ require "optparse"
|
|
4
4
|
module Scaltainer
|
5
5
|
class Command
|
6
6
|
def self.parse(args)
|
7
|
-
configfile, statefile, wait = 'scaltainer.yml', nil, 0
|
7
|
+
configfile, statefile, wait, orchestrator = 'scaltainer.yml', nil, 0, :swarm
|
8
8
|
OptionParser.new do |opts|
|
9
9
|
opts.banner = "Usage: scaltainer [options]"
|
10
10
|
opts.on("-f", "--conf-file FILE", "Specify configuration file (default: scaltainer.yml)") do |file|
|
@@ -16,10 +16,27 @@ module Scaltainer
|
|
16
16
|
opts.on("-w", "--wait SECONDS", "Specify wait time between repeated calls, 0 for no repetition (default: 0)") do |w|
|
17
17
|
wait = w.to_i
|
18
18
|
end
|
19
|
+
opts.on("-o", "--orchestrator swarm:kubernetes", [:swarm, :kubernetes], "Specify orchestrator type (default: swarm)") do |o|
|
20
|
+
orchestrator = o
|
21
|
+
end
|
22
|
+
opts.on("-v", "--version", "Show version and exit") do
|
23
|
+
puts Scaltainer::VERSION
|
24
|
+
exit 0
|
25
|
+
end
|
19
26
|
opts.on_tail("-h", "--help", "Show this message") do
|
20
27
|
puts opts
|
21
28
|
puts "\nEnvironment variables: \n"
|
29
|
+
puts "Docker Swarm options:"
|
22
30
|
puts "- DOCKER_URL: defaults to local socket"
|
31
|
+
puts "Kubernetes options:"
|
32
|
+
puts "- KUBECONFIG: set to Kubernetes config (default: $HOME/.kube/config) if you want to connect to the current configured cluster"
|
33
|
+
puts "- KUBERNETES_API_SERVER: overrides option in KUBECONFIG and defaults to https://kubernetes.default:443"
|
34
|
+
puts "- KUBERNETES_SKIP_SSL_VERIFY: KUBECONFIG option overrides this, set to any value to skip SSL verification"
|
35
|
+
puts "- KUBERNETES_API_ENDPOINT: defaults to /api"
|
36
|
+
puts "- KUBERNETES_API_VERSION: overrides option in KUBECONFIG and defaults to v1"
|
37
|
+
puts "- KUBERNETES_CONTROLLER_KIND: controller kind to scale, allowed values: deployment (default), replication_controller, or replica_set"
|
38
|
+
puts " Make sure the KUBERNETES_CONTROLLER_KIND you specify is part of the api specified using KUBERNETES_API_ENDPOINT and KUBERNETES_API_VERSION"
|
39
|
+
puts "General options:"
|
23
40
|
puts "- HIREFIRE_TOKEN"
|
24
41
|
puts "- NEW_RELIC_API_KEY"
|
25
42
|
puts "- RESPONSE_TIME_WINDOW: defaults to 5"
|
@@ -38,7 +55,7 @@ module Scaltainer
|
|
38
55
|
logger = Logger.new(STDOUT)
|
39
56
|
logger.level = %w(debug info warn error fatal unknown).find_index((ENV['LOG_LEVEL'] || '').downcase) || 1
|
40
57
|
|
41
|
-
return configfile, statefile, logger, wait
|
58
|
+
return configfile, statefile, logger, wait, orchestrator
|
42
59
|
end
|
43
60
|
|
44
61
|
private
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Scaltainer
|
2
|
+
class ReplicaSetBase
|
3
|
+
attr_accessor :id, :name, :type, :namespace
|
4
|
+
|
5
|
+
def initialize(name, type, namespace)
|
6
|
+
@name, @type, @namespace = name, type, namespace
|
7
|
+
end
|
8
|
+
|
9
|
+
def get_replicas
|
10
|
+
raise 'Abstract method, please override'
|
11
|
+
end
|
12
|
+
|
13
|
+
def set_replicas(replicas)
|
14
|
+
raise 'Abstract method, please override'
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'kubeclient'
|
2
|
+
|
3
|
+
module Scaltainer
|
4
|
+
class KubeResource < ReplicaSetBase
|
5
|
+
def initialize(name, namespace)
|
6
|
+
@@client ||= self.class.get_client
|
7
|
+
type = ENV['KUBERNETES_CONTROLLER_KIND'] || 'deployment'
|
8
|
+
# if namespace not specified, use the one found in configuration
|
9
|
+
namespace ||= @@namespace || 'default'
|
10
|
+
super(name, type, namespace)
|
11
|
+
@resource = @@client.send("get_#{@type}", normalize_name(@name), @namespace)
|
12
|
+
@id = @resource.metadata.uid
|
13
|
+
end
|
14
|
+
|
15
|
+
def get_replicas
|
16
|
+
@resource.spec.replicas
|
17
|
+
end
|
18
|
+
|
19
|
+
def set_replicas(replicas)
|
20
|
+
@@client.send("patch_#{@type}", normalize_name(@name), {spec: {replicas: replicas}}, @namespace)
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def self.get_client
|
26
|
+
if ENV['KUBECONFIG']
|
27
|
+
get_client_from_kubeconfig ENV['KUBECONFIG']
|
28
|
+
else
|
29
|
+
get_client_from_serviceaccount '/var/run/secrets/kubernetes.io/serviceaccount'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.get_client_from_kubeconfig(kubeconfig)
|
34
|
+
config = Kubeclient::Config.read(kubeconfig)
|
35
|
+
url = get_api_url(config.context.api_endpoint)
|
36
|
+
version = get_api_version(config.context.api_version)
|
37
|
+
# @@namespace = config.context.namespace # wait till PR#308 merged into kubeclient
|
38
|
+
Kubeclient::Client.new(
|
39
|
+
url, version,
|
40
|
+
ssl_options: config.context.ssl_options,
|
41
|
+
auth_options: config.context.auth_options
|
42
|
+
)
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.get_client_from_serviceaccount(serviceaccount)
|
46
|
+
ssl_verify = if ENV['KUBERNETES_SKIP_SSL_VERIFY']
|
47
|
+
OpenSSL::SSL::VERIFY_NONE
|
48
|
+
else
|
49
|
+
OpenSSL::SSL::VERIFY_PEER
|
50
|
+
end
|
51
|
+
ssl_options = {
|
52
|
+
client_cert: OpenSSL::X509::Certificate.new(read_secret(serviceaccount, 'ca.crt')),
|
53
|
+
verify_ssl: ssl_verify
|
54
|
+
}
|
55
|
+
auth_options = {bearer_token: read_secret(serviceaccount, 'token')}
|
56
|
+
@@namespace = read_secret(serviceaccount, 'namespace')
|
57
|
+
url = get_api_url('https://kubernetes.default:443')
|
58
|
+
version = get_api_version('v1')
|
59
|
+
Kubeclient::Client.new(
|
60
|
+
url, version,
|
61
|
+
ssl_options: ssl_options,
|
62
|
+
auth_options: auth_options
|
63
|
+
)
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.get_api_url(default_server)
|
67
|
+
server = ENV['KUBERNETES_API_SERVER'] || default_server
|
68
|
+
endpoint = ENV['KUBERNETES_API_ENDPOINT'] || '/api'
|
69
|
+
"#{server}#{endpoint}"
|
70
|
+
end
|
71
|
+
|
72
|
+
def self.get_api_version(default_version)
|
73
|
+
ENV['KUBERNETES_API_VERSION'] || default_version
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.read_secret(serviceaccount, secret)
|
77
|
+
File.read("#{serviceaccount}/#{secret}")
|
78
|
+
end
|
79
|
+
|
80
|
+
def normalize_name(name)
|
81
|
+
name.gsub(/_/, '-')
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -1,3 +1,26 @@
|
|
1
|
+
module Scaltainer
|
2
|
+
class DockerService < ReplicaSetBase
|
3
|
+
def initialize(service_name, namespace)
|
4
|
+
# set logger?
|
5
|
+
full_name = namespace ? "#{namespace}_#{service_name}" : service_name
|
6
|
+
@service = Docker::Service.all(filters: {name: [full_name]}.to_json)[0]
|
7
|
+
raise "Docker Service not found: #{full_name}" unless @service
|
8
|
+
@id = @service.id
|
9
|
+
super(service_name, 'service', namespace)
|
10
|
+
end
|
11
|
+
|
12
|
+
def get_replicas
|
13
|
+
replicated = @service.info["Spec"]["Mode"]["Replicated"]
|
14
|
+
raise ConfigurationError.new "Cannot replicate a global service: #{@name}" unless replicated
|
15
|
+
replicated["Replicas"]
|
16
|
+
end
|
17
|
+
|
18
|
+
def set_replicas(replicas)
|
19
|
+
@service.scale replicas
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
1
24
|
# source: https://github.com/Stazer/docker-api/blob/feature/swarm_support/lib/docker/service.rb
|
2
25
|
|
3
26
|
# This class represents a Docker Service. It's important to note that nothing
|
data/lib/scaltainer/runner.rb
CHANGED
@@ -2,7 +2,8 @@ require "yaml"
|
|
2
2
|
|
3
3
|
module Scaltainer
|
4
4
|
class Runner
|
5
|
-
def initialize(configfile, statefile, logger, wait)
|
5
|
+
def initialize(configfile, statefile, logger, wait, orchestrator)
|
6
|
+
@orchestrator = orchestrator
|
6
7
|
@logger = logger
|
7
8
|
@default_service_config = {
|
8
9
|
"min" => 0,
|
@@ -13,7 +14,7 @@ module Scaltainer
|
|
13
14
|
}
|
14
15
|
@logger.debug "Scaltainer initialized with configuration file: #{configfile}, and state file: #{statefile}"
|
15
16
|
config = YAML.load_file configfile
|
16
|
-
Docker.logger = @logger
|
17
|
+
Docker.logger = @logger if orchestrator == :swarm
|
17
18
|
state = get_state(statefile) || {}
|
18
19
|
endpoint = config["endpoint"]
|
19
20
|
service_type_web = ServiceTypeWeb.new(endpoint)
|
@@ -29,9 +30,9 @@ module Scaltainer
|
|
29
30
|
private
|
30
31
|
|
31
32
|
def run(config, state, service_type_web, service_type_worker)
|
32
|
-
|
33
|
-
iterate_services config["web_services"],
|
34
|
-
iterate_services config["worker_services"],
|
33
|
+
namespace = config["namespace"] || config["stack_name"]
|
34
|
+
iterate_services config["web_services"], namespace, service_type_web, state
|
35
|
+
iterate_services config["worker_services"], namespace, service_type_worker, state
|
35
36
|
end
|
36
37
|
|
37
38
|
def get_state(statefile)
|
@@ -42,35 +43,18 @@ module Scaltainer
|
|
42
43
|
File.write(statefile, state.to_yaml)
|
43
44
|
end
|
44
45
|
|
45
|
-
def
|
46
|
-
begin
|
47
|
-
service = Docker::Service.all(filters: {name: [service_name]}.to_json)[0]
|
48
|
-
rescue => e
|
49
|
-
raise NetworkError.new "Could not get service with name #{service_name} from docker engine at #{Docker.url}.\n#{e.message}"
|
50
|
-
end
|
51
|
-
raise ConfigurationError.new "Unknown service to docker: #{service_name}" unless service
|
52
|
-
service
|
53
|
-
end
|
54
|
-
|
55
|
-
def get_service_replicas(service)
|
56
|
-
# ask docker about replicas for service
|
57
|
-
replicated = service.info["Spec"]["Mode"]["Replicated"]
|
58
|
-
raise ConfigurationError.new "Cannot replicate a global service: #{service.info['Spec']['Name']}" unless replicated
|
59
|
-
replicated["Replicas"]
|
60
|
-
end
|
61
|
-
|
62
|
-
def iterate_services(services, service_prefix, type, state)
|
46
|
+
def iterate_services(services, namespace, type, state)
|
63
47
|
begin
|
64
48
|
metrics = type.get_metrics services
|
65
|
-
@logger.debug "Retrieved metrics for #{type}
|
49
|
+
@logger.debug "Retrieved metrics for #{type} resources: #{metrics}"
|
66
50
|
services.each do |service_name, service_config|
|
67
51
|
begin
|
68
52
|
state[service_name] ||= {}
|
69
53
|
service_state = state[service_name]
|
70
|
-
@logger.debug "
|
54
|
+
@logger.debug "Resource #{service_name} in namespace #{namespace} currently has state: #{service_state}"
|
71
55
|
service_config = @default_service_config.merge service_config
|
72
|
-
@logger.debug "
|
73
|
-
process_service service_name, service_config, service_state,
|
56
|
+
@logger.debug "Resource #{service_name} in namespace #{namespace} configuration: #{service_config}"
|
57
|
+
process_service service_name, service_config, service_state, namespace, type, metrics
|
74
58
|
rescue RuntimeError => e
|
75
59
|
# skipping service
|
76
60
|
log_exception e
|
@@ -86,34 +70,46 @@ module Scaltainer
|
|
86
70
|
@logger.log (e.class == Scaltainer::Warning ? Logger::WARN : Logger::ERROR), e.message
|
87
71
|
end
|
88
72
|
|
89
|
-
def process_service(service_name, config, state,
|
90
|
-
|
91
|
-
service
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
raise Scaltainer::Warning.new("Configured service '#{service_name}' not found in metrics endpoint") unless metric
|
73
|
+
def process_service(service_name, config, state, namespace, type, metrics)
|
74
|
+
service = get_service service_name, namespace
|
75
|
+
@logger.debug "Found #{service.type} at orchestrator with name '#{service.name}' and id '#{service.id}'"
|
76
|
+
current_replicas = service.get_replicas
|
77
|
+
@logger.debug "#{service.type.capitalize} #{service.name} is currently configured for #{current_replicas} replica(s)"
|
78
|
+
metric = metrics[service.name]
|
79
|
+
raise Scaltainer::Warning.new("Configured #{service.type} '#{service.name}' not found in metrics endpoint") unless metric
|
97
80
|
desired_replicas = type.determine_desired_replicas metric, config, current_replicas
|
98
|
-
@logger.debug "Desired number of replicas for service #{
|
81
|
+
@logger.debug "Desired number of replicas for #{service.type} #{service.name} is #{desired_replicas}"
|
99
82
|
adjusted_replicas = type.adjust_desired_replicas(desired_replicas, config)
|
100
|
-
@logger.debug "Desired number of replicas for service #{
|
83
|
+
@logger.debug "Desired number of replicas for #{service.type} #{service.name} is adjusted to #{adjusted_replicas}"
|
101
84
|
replica_diff = adjusted_replicas - current_replicas
|
102
85
|
type.yield_to_scale(replica_diff, config, state, metric,
|
103
|
-
|
86
|
+
service.name, @logger) do
|
104
87
|
scale_out service, current_replicas, adjusted_replicas
|
105
88
|
end
|
106
89
|
end
|
107
90
|
|
91
|
+
def get_service(service_name, namespace)
|
92
|
+
begin
|
93
|
+
service = if @orchestrator == :swarm
|
94
|
+
DockerService.new service_name, namespace
|
95
|
+
elsif @orchestrator == :kubernetes
|
96
|
+
KubeResource.new service_name, namespace
|
97
|
+
end
|
98
|
+
rescue => e
|
99
|
+
raise NetworkError.new "Could not find resource with name #{service_name} in namespace #{namespace}: #{e.message}"
|
100
|
+
end
|
101
|
+
raise ConfigurationError.new "Unknown resource: #{service_name} in namespace #{namespace}" unless service
|
102
|
+
service
|
103
|
+
end
|
104
|
+
|
108
105
|
def scale_out(service, current_replicas, desired_replicas)
|
109
106
|
return if current_replicas == desired_replicas
|
110
|
-
# send scale command to
|
111
|
-
|
112
|
-
@logger.info "Scaling #{service_name} from #{current_replicas} to #{desired_replicas}"
|
107
|
+
# send scale command to orchestrator
|
108
|
+
@logger.info "Scaling #{service.type} #{service.name} from #{current_replicas} to #{desired_replicas}"
|
113
109
|
begin
|
114
|
-
service.
|
110
|
+
service.set_replicas desired_replicas
|
115
111
|
rescue => e
|
116
|
-
raise NetworkError.new "Could not scale service #{
|
112
|
+
raise NetworkError.new "Could not scale #{service.type} #{service.name} due to error: #{e.message}"
|
117
113
|
end
|
118
114
|
end
|
119
115
|
|
@@ -6,12 +6,12 @@ module Scaltainer
|
|
6
6
|
|
7
7
|
def get_metrics(services)
|
8
8
|
services_count = services.keys.length rescue 0
|
9
|
-
raise Scaltainer::Warning.new "No
|
9
|
+
raise Scaltainer::Warning.new "No resources found for #{self.class.name}" if services_count == 0
|
10
10
|
end
|
11
11
|
|
12
12
|
def determine_desired_replicas(metric, service_config, current_replicas)
|
13
|
-
raise ConfigurationError.new 'No metric found for requested
|
14
|
-
raise ConfigurationError.new 'No configuration found for requested
|
13
|
+
raise ConfigurationError.new 'No metric found for requested resource' unless metric
|
14
|
+
raise ConfigurationError.new 'No configuration found for requested resource' unless service_config
|
15
15
|
end
|
16
16
|
|
17
17
|
def adjust_desired_replicas(desired_replicas, config)
|
@@ -32,7 +32,7 @@ module Scaltainer
|
|
32
32
|
yield
|
33
33
|
state["upscale_sensitivity"] = 0
|
34
34
|
else
|
35
|
-
logger.debug "Scaling up of
|
35
|
+
logger.debug "Scaling up of resource #{service_name} blocked by upscale_sensitivity at level " +
|
36
36
|
"#{state["upscale_sensitivity"]} while level #{config["upscale_sensitivity"]} is required"
|
37
37
|
end
|
38
38
|
elsif replica_diff < 0 # TODO force down when above max?
|
@@ -45,17 +45,17 @@ module Scaltainer
|
|
45
45
|
yield
|
46
46
|
state["downscale_sensitivity"] = 0
|
47
47
|
else
|
48
|
-
logger.debug "Scaling down of
|
48
|
+
logger.debug "Scaling down of resource #{service_name} blocked by downscale_sensitivity at level " +
|
49
49
|
"#{state["downscale_sensitivity"]} while level #{config["downscale_sensitivity"]} is required"
|
50
50
|
end
|
51
51
|
else
|
52
|
-
logger.debug "Scaling down of
|
52
|
+
logger.debug "Scaling down of resource #{service_name} to #{metric} replicas blocked by a non-decrementable config"
|
53
53
|
end
|
54
54
|
else
|
55
55
|
# no breach, change state
|
56
56
|
state["upscale_sensitivity"] = 0
|
57
57
|
state["downscale_sensitivity"] = 0
|
58
|
-
logger.info "No need to scale
|
58
|
+
logger.info "No need to scale resource #{service_name}"
|
59
59
|
end
|
60
60
|
end
|
61
61
|
|
@@ -14,7 +14,7 @@ module Scaltainer
|
|
14
14
|
|
15
15
|
services.reduce({}) do |hash, (service_name, service_config)|
|
16
16
|
app_id = service_config["newrelic_app_id"]
|
17
|
-
raise ConfigurationError.new "
|
17
|
+
raise ConfigurationError.new "Resource #{service_name} does not have a corresponding newrelic_app_id" unless app_id
|
18
18
|
|
19
19
|
begin
|
20
20
|
metric = nr.get_avg_response_time app_id, from, to
|
@@ -28,8 +28,8 @@ module Scaltainer
|
|
28
28
|
|
29
29
|
def determine_desired_replicas(metric, service_config, current_replicas)
|
30
30
|
super
|
31
|
-
raise ConfigurationError.new "Missing max_response_time in web
|
32
|
-
raise ConfigurationError.new "Missing min_response_time in web
|
31
|
+
raise ConfigurationError.new "Missing max_response_time in web resource configuration" unless service_config["max_response_time"]
|
32
|
+
raise ConfigurationError.new "Missing min_response_time in web resource configuration" unless service_config["min_response_time"]
|
33
33
|
unless service_config["min_response_time"] <= service_config["max_response_time"]
|
34
34
|
raise ConfigurationError.new "min_response_time and max_response_time are not in order"
|
35
35
|
end
|
@@ -21,7 +21,7 @@ module Scaltainer
|
|
21
21
|
|
22
22
|
def determine_desired_replicas(metric, service_config, current_replicas)
|
23
23
|
super
|
24
|
-
raise ConfigurationError.new "Missing ratio in worker
|
24
|
+
raise ConfigurationError.new "Missing ratio in worker resource configuration" unless service_config["ratio"]
|
25
25
|
if !metric.is_a?(Integer) || metric < 0
|
26
26
|
raise ConfigurationError.new "#{metric} is an invalid metric value, must be a non-negative number"
|
27
27
|
end
|
data/lib/scaltainer/version.rb
CHANGED
data/scaltainer.gemspec
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: scaltainer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hossam Hammady
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-03-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -108,6 +108,20 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: kubeclient
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
111
125
|
- !ruby/object:Gem::Dependency
|
112
126
|
name: dotenv
|
113
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -132,6 +146,7 @@ executables:
|
|
132
146
|
extensions: []
|
133
147
|
extra_rdoc_files: []
|
134
148
|
files:
|
149
|
+
- ".dockerignore"
|
135
150
|
- ".gitignore"
|
136
151
|
- ".rspec"
|
137
152
|
- ".travis.yml"
|
@@ -145,9 +160,12 @@ files:
|
|
145
160
|
- exe/scaltainer
|
146
161
|
- lib/scaltainer.rb
|
147
162
|
- lib/scaltainer/command.rb
|
148
|
-
- lib/scaltainer/docker/service.rb
|
149
163
|
- lib/scaltainer/exceptions.rb
|
150
164
|
- lib/scaltainer/newrelic/metrics.rb
|
165
|
+
- lib/scaltainer/orchestrators.rb
|
166
|
+
- lib/scaltainer/orchestrators/base.rb
|
167
|
+
- lib/scaltainer/orchestrators/kubernetes.rb
|
168
|
+
- lib/scaltainer/orchestrators/swarm.rb
|
151
169
|
- lib/scaltainer/runner.rb
|
152
170
|
- lib/scaltainer/service_types.rb
|
153
171
|
- lib/scaltainer/service_types/base.rb
|