scaltainer 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.dockerignore +3 -0
- data/.gitignore +5 -1
- data/Dockerfile +4 -2
- data/README.md +124 -32
- data/exe/scaltainer +2 -2
- data/lib/scaltainer.rb +1 -1
- data/lib/scaltainer/command.rb +19 -2
- data/lib/scaltainer/orchestrators.rb +3 -0
- data/lib/scaltainer/orchestrators/base.rb +17 -0
- data/lib/scaltainer/orchestrators/kubernetes.rb +84 -0
- data/lib/scaltainer/{docker/service.rb → orchestrators/swarm.rb} +23 -0
- data/lib/scaltainer/runner.rb +39 -43
- data/lib/scaltainer/service_types/base.rb +7 -7
- data/lib/scaltainer/service_types/web.rb +3 -3
- data/lib/scaltainer/service_types/worker.rb +1 -1
- data/lib/scaltainer/version.rb +1 -1
- data/scaltainer.gemspec +1 -0
- metadata +21 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 642f49828031f354f271d0ad4e28268ab40e1f1a
|
4
|
+
data.tar.gz: 2be74e2694d99bc5edde992e8e277867db0de44b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 34a7ddda22e20d4e909a29c7b2183344712a7fc72dd8d970055c578834e6d1707496bfeebfd09cc4224d62a806615756cb964bddb07653b60798ded0ffe0cbe8
|
7
|
+
data.tar.gz: 07b3b919496e8cc1fbdbc25199e53509e5bb44b3be47a0257430ce6be22102dc08eb43e7d6a2e9bc2eb6ef2ad8d0d33b84ebec9a5c4be8ee83b5094df7e25bc5
|
data/.dockerignore
ADDED
data/.gitignore
CHANGED
data/Dockerfile
CHANGED
@@ -1,8 +1,10 @@
|
|
1
1
|
FROM ruby:2.3
|
2
2
|
|
3
|
-
|
3
|
+
label maintainer="Hossam Hammady <github@hammady.net>"
|
4
4
|
|
5
|
-
|
5
|
+
WORKDIR /home
|
6
|
+
COPY / /home/
|
7
|
+
RUN bundle install && bundle exec rake install
|
6
8
|
|
7
9
|
ENTRYPOINT ["scaltainer"]
|
8
10
|
|
data/README.md
CHANGED
@@ -4,32 +4,38 @@
|
|
4
4
|
|
5
5
|
# Scaltainer
|
6
6
|
|
7
|
-
A Ruby gem to monitor
|
7
|
+
A Ruby gem to monitor Docker Swarm mode services and Kubernetes resources
|
8
|
+
and auto-scale them based on user configuration.
|
8
9
|
It can be used to monitor web services and worker services. The web services type has metrics like response time using [New Relic](https://newrelic.com/). The worker services type metrics are basically the queue size for each.
|
9
10
|
This gem is inspired by [HireFire](https://manager.hirefire.io/) and was indeed motivated by the migration
|
10
|
-
from [Heroku](https://www.heroku.com/) to Docker
|
11
|
+
from [Heroku](https://www.heroku.com/) to Docker.
|
11
12
|
|
12
13
|
## Installation
|
13
14
|
|
14
15
|
Add this line to your application's Gemfile:
|
15
16
|
|
16
|
-
|
17
|
-
gem 'scaltainer'
|
18
|
-
```
|
17
|
+
Install using rubygems:
|
19
18
|
|
20
|
-
|
19
|
+
$ gem install scaltainer
|
21
20
|
|
22
|
-
|
21
|
+
## Usage
|
23
22
|
|
24
|
-
|
23
|
+
For Docker swarm:
|
25
24
|
|
26
|
-
|
25
|
+
scaltainer -o swarm
|
27
26
|
|
28
|
-
|
27
|
+
Or simply:
|
29
28
|
|
30
29
|
scaltainer
|
31
30
|
|
32
|
-
|
31
|
+
For Kubernetes:
|
32
|
+
|
33
|
+
scaltainer -o kubernetes
|
34
|
+
|
35
|
+
|
36
|
+
This will do a one-time check on the running docker service replicas
|
37
|
+
or Kubernetes replication controllers, replica sets, or deployments.
|
38
|
+
Then it sends scaling out/in commands to the cluster as appropriate.
|
33
39
|
Configuration is read from `scaltainer.yml` by default. If you want to read from another file add `-f yourconfig.yml`:
|
34
40
|
|
35
41
|
scaltainer -f yourconfig.yml
|
@@ -47,15 +53,44 @@ specify the wait time between repetitions using the `-w` parameter in seconds:
|
|
47
53
|
|
48
54
|
scaltainer -w 60
|
49
55
|
|
50
|
-
This will repeatedly call scaltainer every 60 seconds, sleeping in
|
56
|
+
This will repeatedly call scaltainer every 60 seconds, sleeping in-between.
|
51
57
|
|
52
58
|
## Configuration
|
53
59
|
|
54
60
|
### Environment variables
|
55
61
|
|
62
|
+
#### Docker swarm options
|
63
|
+
|
56
64
|
- `DOCKER_URL`: Should point to the docker engine URL.
|
57
65
|
If not set, it defaults to local unix socket.
|
58
66
|
|
67
|
+
#### Kubernetes options
|
68
|
+
|
69
|
+
- `KUBECONFIG`: set to Kubernetes config
|
70
|
+
(default: `$HOME/.kube/config`) if you want to connect
|
71
|
+
to the current configured cluster.
|
72
|
+
|
73
|
+
- `KUBERNETES_API_SERVER`: overrides option in `KUBECONFIG`
|
74
|
+
and defaults to `https://kubernetes.default:443`.
|
75
|
+
|
76
|
+
- `KUBERNETES_SKIP_SSL_VERIFY`: `KUBECONFIG` option overrides
|
77
|
+
this, set to any value to skip SSL verification.
|
78
|
+
|
79
|
+
- `KUBERNETES_API_ENDPOINT`: defaults to `/api`.
|
80
|
+
|
81
|
+
- `KUBERNETES_API_VERSION`: overrides option in `KUBECONFIG`
|
82
|
+
and defaults to `v1`.
|
83
|
+
|
84
|
+
- `KUBERNETES_CONTROLLER_KIND`: controller kind to scale,
|
85
|
+
allowed values: `deployment` (default),
|
86
|
+
`replication_controller`, or `replica_set`.
|
87
|
+
|
88
|
+
Make sure the `KUBERNETES_CONTROLLER_KIND` you specify is
|
89
|
+
part of the api specified using `KUBERNETES_API_ENDPOINT`
|
90
|
+
and `KUBERNETES_API_VERSION`.
|
91
|
+
|
92
|
+
#### General options
|
93
|
+
|
59
94
|
- `HIREFIRE_TOKEN`: If your application is configured the
|
60
95
|
[hirefire](https://help.hirefire.io/guides/hirefire/job-queue-any-programming-language) way, you need to
|
61
96
|
set `HIREFIRE_TOKEN` environment variable before invoking
|
@@ -86,8 +121,8 @@ The configuration file (determined by `-f FILE` command line parameter) should b
|
|
86
121
|
|
87
122
|
# to get worker metrics
|
88
123
|
endpoint: https://your-app.com/hirefire/$HIREFIRE_TOKEN/info
|
89
|
-
# optional docker swarm stack name
|
90
|
-
|
124
|
+
# optional docker swarm stack name or kubernetes namespace
|
125
|
+
namespace: mynamespace
|
91
126
|
# list of web services to monitor
|
92
127
|
web_services:
|
93
128
|
# each service name should match docker service name
|
@@ -126,25 +161,15 @@ The configuration file (determined by `-f FILE` command line parameter) should b
|
|
126
161
|
|
127
162
|
More details about configuration parameters can be found in [HireFire docs](https://help.hirefire.io/guides).
|
128
163
|
|
129
|
-
## Docker
|
164
|
+
## Docker Swarm usage
|
130
165
|
|
131
|
-
|
132
|
-
|
133
|
-
docker run -it --rm rayyanqcri/scaltainer
|
134
|
-
|
135
|
-
Which will print the usage. To add arguments, just append them:
|
136
|
-
|
137
|
-
docker run -it --rm rayyanqcri/scaltainer -f scaltainer.yml
|
138
|
-
|
139
|
-
Scaltainer should typically be run as a minutely cron service.
|
140
|
-
If you are using [rayyanqcri/swarm-scheduler](https://github.com/rayyanqcri/swarm-scheduler),
|
141
|
-
a service definition for scaltainer is typically something like this:
|
166
|
+
A service definition for scaltainer is typically something like this:
|
142
167
|
|
143
168
|
version: '3.3'
|
144
169
|
services:
|
145
170
|
scaltainer:
|
146
171
|
image: rayyanqcri/scaltainer:latest
|
147
|
-
command: -f /scaltainer.yml --state-file /tmp/scaltainer-state.yml
|
172
|
+
command: -f /scaltainer.yml --state-file /tmp/scaltainer-state.yml -w 60
|
148
173
|
volumes:
|
149
174
|
- /var/run/docker.sock:/var/run/docker.sock
|
150
175
|
environment:
|
@@ -157,9 +182,7 @@ a service definition for scaltainer is typically something like this:
|
|
157
182
|
secrets:
|
158
183
|
- scaltainer
|
159
184
|
deploy:
|
160
|
-
replicas:
|
161
|
-
restart_policy:
|
162
|
-
condition: none
|
185
|
+
replicas: 1
|
163
186
|
placement:
|
164
187
|
constraints:
|
165
188
|
- node.role == manager
|
@@ -177,6 +200,75 @@ Where `scaltainer.env` is a file containing HireFire and NewRelic secrets:
|
|
177
200
|
|
178
201
|
And `scaltainer.yml` is the scaltainer configuration file.
|
179
202
|
|
203
|
+
## Kubernetes usage
|
204
|
+
|
205
|
+
### Create a ConfigMap
|
206
|
+
|
207
|
+
kubectl create configmap scaltainer --from-file=scaltainer.yaml=/path/to/your/scaltainer.yml
|
208
|
+
|
209
|
+
Where `/path/to/your/scaltainer.yml` is the scaltainer configuration file.
|
210
|
+
|
211
|
+
### Create a Secret
|
212
|
+
|
213
|
+
kubectl create secret generic scaltainer --from-env-file=/path/to/scaltainer.env
|
214
|
+
|
215
|
+
Where `/path/to/scaltainer.env` is a file containing HireFire and NewRelic secrets:
|
216
|
+
|
217
|
+
HIREFIRE_TOKEN=
|
218
|
+
NEW_RELIC_API_KEY=
|
219
|
+
|
220
|
+
### Create a Deployment:
|
221
|
+
|
222
|
+
kubectl apply -f scaltainer-kube.yaml
|
223
|
+
|
224
|
+
Where scaltainer-kube.yaml has the following content:
|
225
|
+
|
226
|
+
apiVersion: extensions/v1beta1
|
227
|
+
kind: Deployment
|
228
|
+
metadata:
|
229
|
+
labels:
|
230
|
+
app: scaltainer
|
231
|
+
name: scaltainer
|
232
|
+
spec:
|
233
|
+
replicas: 1
|
234
|
+
template:
|
235
|
+
metadata:
|
236
|
+
labels:
|
237
|
+
app: scaltainer
|
238
|
+
spec:
|
239
|
+
containers:
|
240
|
+
- image: rayyanqcri/scaltainer:latest
|
241
|
+
name: scaltainer
|
242
|
+
args:
|
243
|
+
- -o
|
244
|
+
- kubernetes
|
245
|
+
- -f
|
246
|
+
- /etc/config/scaltainer.yaml
|
247
|
+
- --state-file
|
248
|
+
- /tmp/scaltainer-state.yaml
|
249
|
+
- -w
|
250
|
+
- "60"
|
251
|
+
env:
|
252
|
+
- name: KUBERNETES_SKIP_SSL_VERIFY
|
253
|
+
value: "yes"
|
254
|
+
- name: KUBERNETES_API_ENDPOINT
|
255
|
+
value: /apis/extensions
|
256
|
+
- name: KUBERNETES_API_VERSION
|
257
|
+
value: v1beta1
|
258
|
+
- name: KUBERNETES_CONTROLLER_KIND
|
259
|
+
value: deployment
|
260
|
+
envFrom:
|
261
|
+
- secretRef:
|
262
|
+
name: scaltainer
|
263
|
+
volumeMounts:
|
264
|
+
- name: scaltainer-config
|
265
|
+
mountPath: "/etc/config"
|
266
|
+
volumes:
|
267
|
+
- name: scaltainer-config
|
268
|
+
configMap:
|
269
|
+
name: scaltainer
|
270
|
+
|
271
|
+
|
180
272
|
## Development
|
181
273
|
|
182
274
|
After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
@@ -187,9 +279,9 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
|
|
187
279
|
|
188
280
|
Bug reports and pull requests are welcome on GitHub at https://github.com/hammady/scaltainer.
|
189
281
|
|
190
|
-
##
|
282
|
+
## Testing
|
191
283
|
|
192
|
-
|
284
|
+
rake
|
193
285
|
|
194
286
|
## License
|
195
287
|
|
data/exe/scaltainer
CHANGED
@@ -3,8 +3,8 @@
|
|
3
3
|
require 'scaltainer'
|
4
4
|
|
5
5
|
begin
|
6
|
-
configfile, statefile, logger, wait = Scaltainer::Command.parse ARGV
|
7
|
-
Scaltainer::Runner.new configfile, statefile, logger, wait
|
6
|
+
configfile, statefile, logger, wait, orchestrator = Scaltainer::Command.parse ARGV
|
7
|
+
Scaltainer::Runner.new configfile, statefile, logger, wait, orchestrator
|
8
8
|
rescue => e
|
9
9
|
$stderr.puts e.message
|
10
10
|
$stderr.puts e.backtrace
|
data/lib/scaltainer.rb
CHANGED
data/lib/scaltainer/command.rb
CHANGED
@@ -4,7 +4,7 @@ require "optparse"
|
|
4
4
|
module Scaltainer
|
5
5
|
class Command
|
6
6
|
def self.parse(args)
|
7
|
-
configfile, statefile, wait = 'scaltainer.yml', nil, 0
|
7
|
+
configfile, statefile, wait, orchestrator = 'scaltainer.yml', nil, 0, :swarm
|
8
8
|
OptionParser.new do |opts|
|
9
9
|
opts.banner = "Usage: scaltainer [options]"
|
10
10
|
opts.on("-f", "--conf-file FILE", "Specify configuration file (default: scaltainer.yml)") do |file|
|
@@ -16,10 +16,27 @@ module Scaltainer
|
|
16
16
|
opts.on("-w", "--wait SECONDS", "Specify wait time between repeated calls, 0 for no repetition (default: 0)") do |w|
|
17
17
|
wait = w.to_i
|
18
18
|
end
|
19
|
+
opts.on("-o", "--orchestrator swarm:kubernetes", [:swarm, :kubernetes], "Specify orchestrator type (default: swarm)") do |o|
|
20
|
+
orchestrator = o
|
21
|
+
end
|
22
|
+
opts.on("-v", "--version", "Show version and exit") do
|
23
|
+
puts Scaltainer::VERSION
|
24
|
+
exit 0
|
25
|
+
end
|
19
26
|
opts.on_tail("-h", "--help", "Show this message") do
|
20
27
|
puts opts
|
21
28
|
puts "\nEnvironment variables: \n"
|
29
|
+
puts "Docker Swarm options:"
|
22
30
|
puts "- DOCKER_URL: defaults to local socket"
|
31
|
+
puts "Kubernetes options:"
|
32
|
+
puts "- KUBECONFIG: set to Kubernetes config (default: $HOME/.kube/config) if you want to connect to the current configured cluster"
|
33
|
+
puts "- KUBERNETES_API_SERVER: overrides option in KUBECONFIG and defaults to https://kubernetes.default:443"
|
34
|
+
puts "- KUBERNETES_SKIP_SSL_VERIFY: KUBECONFIG option overrides this, set to any value to skip SSL verification"
|
35
|
+
puts "- KUBERNETES_API_ENDPOINT: defaults to /api"
|
36
|
+
puts "- KUBERNETES_API_VERSION: overrides option in KUBECONFIG and defaults to v1"
|
37
|
+
puts "- KUBERNETES_CONTROLLER_KIND: controller kind to scale, allowed values: deployment (default), replication_controller, or replica_set"
|
38
|
+
puts " Make sure the KUBERNETES_CONTROLLER_KIND you specify is part of the api specified using KUBERNETES_API_ENDPOINT and KUBERNETES_API_VERSION"
|
39
|
+
puts "General options:"
|
23
40
|
puts "- HIREFIRE_TOKEN"
|
24
41
|
puts "- NEW_RELIC_API_KEY"
|
25
42
|
puts "- RESPONSE_TIME_WINDOW: defaults to 5"
|
@@ -38,7 +55,7 @@ module Scaltainer
|
|
38
55
|
logger = Logger.new(STDOUT)
|
39
56
|
logger.level = %w(debug info warn error fatal unknown).find_index((ENV['LOG_LEVEL'] || '').downcase) || 1
|
40
57
|
|
41
|
-
return configfile, statefile, logger, wait
|
58
|
+
return configfile, statefile, logger, wait, orchestrator
|
42
59
|
end
|
43
60
|
|
44
61
|
private
|
@@ -0,0 +1,17 @@
|
|
1
|
+
module Scaltainer
|
2
|
+
class ReplicaSetBase
|
3
|
+
attr_accessor :id, :name, :type, :namespace
|
4
|
+
|
5
|
+
def initialize(name, type, namespace)
|
6
|
+
@name, @type, @namespace = name, type, namespace
|
7
|
+
end
|
8
|
+
|
9
|
+
def get_replicas
|
10
|
+
raise 'Abstract method, please override'
|
11
|
+
end
|
12
|
+
|
13
|
+
def set_replicas(replicas)
|
14
|
+
raise 'Abstract method, please override'
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'kubeclient'
|
2
|
+
|
3
|
+
module Scaltainer
|
4
|
+
class KubeResource < ReplicaSetBase
|
5
|
+
def initialize(name, namespace)
|
6
|
+
@@client ||= self.class.get_client
|
7
|
+
type = ENV['KUBERNETES_CONTROLLER_KIND'] || 'deployment'
|
8
|
+
# if namespace not specified, use the one found in configuration
|
9
|
+
namespace ||= @@namespace || 'default'
|
10
|
+
super(name, type, namespace)
|
11
|
+
@resource = @@client.send("get_#{@type}", normalize_name(@name), @namespace)
|
12
|
+
@id = @resource.metadata.uid
|
13
|
+
end
|
14
|
+
|
15
|
+
def get_replicas
|
16
|
+
@resource.spec.replicas
|
17
|
+
end
|
18
|
+
|
19
|
+
def set_replicas(replicas)
|
20
|
+
@@client.send("patch_#{@type}", normalize_name(@name), {spec: {replicas: replicas}}, @namespace)
|
21
|
+
end
|
22
|
+
|
23
|
+
private
|
24
|
+
|
25
|
+
def self.get_client
|
26
|
+
if ENV['KUBECONFIG']
|
27
|
+
get_client_from_kubeconfig ENV['KUBECONFIG']
|
28
|
+
else
|
29
|
+
get_client_from_serviceaccount '/var/run/secrets/kubernetes.io/serviceaccount'
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def self.get_client_from_kubeconfig(kubeconfig)
|
34
|
+
config = Kubeclient::Config.read(kubeconfig)
|
35
|
+
url = get_api_url(config.context.api_endpoint)
|
36
|
+
version = get_api_version(config.context.api_version)
|
37
|
+
# @@namespace = config.context.namespace # wait till PR#308 merged into kubeclient
|
38
|
+
Kubeclient::Client.new(
|
39
|
+
url, version,
|
40
|
+
ssl_options: config.context.ssl_options,
|
41
|
+
auth_options: config.context.auth_options
|
42
|
+
)
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.get_client_from_serviceaccount(serviceaccount)
|
46
|
+
ssl_verify = if ENV['KUBERNETES_SKIP_SSL_VERIFY']
|
47
|
+
OpenSSL::SSL::VERIFY_NONE
|
48
|
+
else
|
49
|
+
OpenSSL::SSL::VERIFY_PEER
|
50
|
+
end
|
51
|
+
ssl_options = {
|
52
|
+
client_cert: OpenSSL::X509::Certificate.new(read_secret(serviceaccount, 'ca.crt')),
|
53
|
+
verify_ssl: ssl_verify
|
54
|
+
}
|
55
|
+
auth_options = {bearer_token: read_secret(serviceaccount, 'token')}
|
56
|
+
@@namespace = read_secret(serviceaccount, 'namespace')
|
57
|
+
url = get_api_url('https://kubernetes.default:443')
|
58
|
+
version = get_api_version('v1')
|
59
|
+
Kubeclient::Client.new(
|
60
|
+
url, version,
|
61
|
+
ssl_options: ssl_options,
|
62
|
+
auth_options: auth_options
|
63
|
+
)
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.get_api_url(default_server)
|
67
|
+
server = ENV['KUBERNETES_API_SERVER'] || default_server
|
68
|
+
endpoint = ENV['KUBERNETES_API_ENDPOINT'] || '/api'
|
69
|
+
"#{server}#{endpoint}"
|
70
|
+
end
|
71
|
+
|
72
|
+
def self.get_api_version(default_version)
|
73
|
+
ENV['KUBERNETES_API_VERSION'] || default_version
|
74
|
+
end
|
75
|
+
|
76
|
+
def self.read_secret(serviceaccount, secret)
|
77
|
+
File.read("#{serviceaccount}/#{secret}")
|
78
|
+
end
|
79
|
+
|
80
|
+
def normalize_name(name)
|
81
|
+
name.gsub(/_/, '-')
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
@@ -1,3 +1,26 @@
|
|
1
|
+
module Scaltainer
|
2
|
+
class DockerService < ReplicaSetBase
|
3
|
+
def initialize(service_name, namespace)
|
4
|
+
# set logger?
|
5
|
+
full_name = namespace ? "#{namespace}_#{service_name}" : service_name
|
6
|
+
@service = Docker::Service.all(filters: {name: [full_name]}.to_json)[0]
|
7
|
+
raise "Docker Service not found: #{full_name}" unless @service
|
8
|
+
@id = @service.id
|
9
|
+
super(service_name, 'service', namespace)
|
10
|
+
end
|
11
|
+
|
12
|
+
def get_replicas
|
13
|
+
replicated = @service.info["Spec"]["Mode"]["Replicated"]
|
14
|
+
raise ConfigurationError.new "Cannot replicate a global service: #{@name}" unless replicated
|
15
|
+
replicated["Replicas"]
|
16
|
+
end
|
17
|
+
|
18
|
+
def set_replicas(replicas)
|
19
|
+
@service.scale replicas
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
1
24
|
# source: https://github.com/Stazer/docker-api/blob/feature/swarm_support/lib/docker/service.rb
|
2
25
|
|
3
26
|
# This class represents a Docker Service. It's important to note that nothing
|
data/lib/scaltainer/runner.rb
CHANGED
@@ -2,7 +2,8 @@ require "yaml"
|
|
2
2
|
|
3
3
|
module Scaltainer
|
4
4
|
class Runner
|
5
|
-
def initialize(configfile, statefile, logger, wait)
|
5
|
+
def initialize(configfile, statefile, logger, wait, orchestrator)
|
6
|
+
@orchestrator = orchestrator
|
6
7
|
@logger = logger
|
7
8
|
@default_service_config = {
|
8
9
|
"min" => 0,
|
@@ -13,7 +14,7 @@ module Scaltainer
|
|
13
14
|
}
|
14
15
|
@logger.debug "Scaltainer initialized with configuration file: #{configfile}, and state file: #{statefile}"
|
15
16
|
config = YAML.load_file configfile
|
16
|
-
Docker.logger = @logger
|
17
|
+
Docker.logger = @logger if orchestrator == :swarm
|
17
18
|
state = get_state(statefile) || {}
|
18
19
|
endpoint = config["endpoint"]
|
19
20
|
service_type_web = ServiceTypeWeb.new(endpoint)
|
@@ -29,9 +30,9 @@ module Scaltainer
|
|
29
30
|
private
|
30
31
|
|
31
32
|
def run(config, state, service_type_web, service_type_worker)
|
32
|
-
|
33
|
-
iterate_services config["web_services"],
|
34
|
-
iterate_services config["worker_services"],
|
33
|
+
namespace = config["namespace"] || config["stack_name"]
|
34
|
+
iterate_services config["web_services"], namespace, service_type_web, state
|
35
|
+
iterate_services config["worker_services"], namespace, service_type_worker, state
|
35
36
|
end
|
36
37
|
|
37
38
|
def get_state(statefile)
|
@@ -42,35 +43,18 @@ module Scaltainer
|
|
42
43
|
File.write(statefile, state.to_yaml)
|
43
44
|
end
|
44
45
|
|
45
|
-
def
|
46
|
-
begin
|
47
|
-
service = Docker::Service.all(filters: {name: [service_name]}.to_json)[0]
|
48
|
-
rescue => e
|
49
|
-
raise NetworkError.new "Could not get service with name #{service_name} from docker engine at #{Docker.url}.\n#{e.message}"
|
50
|
-
end
|
51
|
-
raise ConfigurationError.new "Unknown service to docker: #{service_name}" unless service
|
52
|
-
service
|
53
|
-
end
|
54
|
-
|
55
|
-
def get_service_replicas(service)
|
56
|
-
# ask docker about replicas for service
|
57
|
-
replicated = service.info["Spec"]["Mode"]["Replicated"]
|
58
|
-
raise ConfigurationError.new "Cannot replicate a global service: #{service.info['Spec']['Name']}" unless replicated
|
59
|
-
replicated["Replicas"]
|
60
|
-
end
|
61
|
-
|
62
|
-
def iterate_services(services, service_prefix, type, state)
|
46
|
+
def iterate_services(services, namespace, type, state)
|
63
47
|
begin
|
64
48
|
metrics = type.get_metrics services
|
65
|
-
@logger.debug "Retrieved metrics for #{type}
|
49
|
+
@logger.debug "Retrieved metrics for #{type} resources: #{metrics}"
|
66
50
|
services.each do |service_name, service_config|
|
67
51
|
begin
|
68
52
|
state[service_name] ||= {}
|
69
53
|
service_state = state[service_name]
|
70
|
-
@logger.debug "
|
54
|
+
@logger.debug "Resource #{service_name} in namespace #{namespace} currently has state: #{service_state}"
|
71
55
|
service_config = @default_service_config.merge service_config
|
72
|
-
@logger.debug "
|
73
|
-
process_service service_name, service_config, service_state,
|
56
|
+
@logger.debug "Resource #{service_name} in namespace #{namespace} configuration: #{service_config}"
|
57
|
+
process_service service_name, service_config, service_state, namespace, type, metrics
|
74
58
|
rescue RuntimeError => e
|
75
59
|
# skipping service
|
76
60
|
log_exception e
|
@@ -86,34 +70,46 @@ module Scaltainer
|
|
86
70
|
@logger.log (e.class == Scaltainer::Warning ? Logger::WARN : Logger::ERROR), e.message
|
87
71
|
end
|
88
72
|
|
89
|
-
def process_service(service_name, config, state,
|
90
|
-
|
91
|
-
service
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
raise Scaltainer::Warning.new("Configured service '#{service_name}' not found in metrics endpoint") unless metric
|
73
|
+
def process_service(service_name, config, state, namespace, type, metrics)
|
74
|
+
service = get_service service_name, namespace
|
75
|
+
@logger.debug "Found #{service.type} at orchestrator with name '#{service.name}' and id '#{service.id}'"
|
76
|
+
current_replicas = service.get_replicas
|
77
|
+
@logger.debug "#{service.type.capitalize} #{service.name} is currently configured for #{current_replicas} replica(s)"
|
78
|
+
metric = metrics[service.name]
|
79
|
+
raise Scaltainer::Warning.new("Configured #{service.type} '#{service.name}' not found in metrics endpoint") unless metric
|
97
80
|
desired_replicas = type.determine_desired_replicas metric, config, current_replicas
|
98
|
-
@logger.debug "Desired number of replicas for service #{
|
81
|
+
@logger.debug "Desired number of replicas for #{service.type} #{service.name} is #{desired_replicas}"
|
99
82
|
adjusted_replicas = type.adjust_desired_replicas(desired_replicas, config)
|
100
|
-
@logger.debug "Desired number of replicas for service #{
|
83
|
+
@logger.debug "Desired number of replicas for #{service.type} #{service.name} is adjusted to #{adjusted_replicas}"
|
101
84
|
replica_diff = adjusted_replicas - current_replicas
|
102
85
|
type.yield_to_scale(replica_diff, config, state, metric,
|
103
|
-
|
86
|
+
service.name, @logger) do
|
104
87
|
scale_out service, current_replicas, adjusted_replicas
|
105
88
|
end
|
106
89
|
end
|
107
90
|
|
91
|
+
def get_service(service_name, namespace)
|
92
|
+
begin
|
93
|
+
service = if @orchestrator == :swarm
|
94
|
+
DockerService.new service_name, namespace
|
95
|
+
elsif @orchestrator == :kubernetes
|
96
|
+
KubeResource.new service_name, namespace
|
97
|
+
end
|
98
|
+
rescue => e
|
99
|
+
raise NetworkError.new "Could not find resource with name #{service_name} in namespace #{namespace}: #{e.message}"
|
100
|
+
end
|
101
|
+
raise ConfigurationError.new "Unknown resource: #{service_name} in namespace #{namespace}" unless service
|
102
|
+
service
|
103
|
+
end
|
104
|
+
|
108
105
|
def scale_out(service, current_replicas, desired_replicas)
|
109
106
|
return if current_replicas == desired_replicas
|
110
|
-
# send scale command to
|
111
|
-
|
112
|
-
@logger.info "Scaling #{service_name} from #{current_replicas} to #{desired_replicas}"
|
107
|
+
# send scale command to orchestrator
|
108
|
+
@logger.info "Scaling #{service.type} #{service.name} from #{current_replicas} to #{desired_replicas}"
|
113
109
|
begin
|
114
|
-
service.
|
110
|
+
service.set_replicas desired_replicas
|
115
111
|
rescue => e
|
116
|
-
raise NetworkError.new "Could not scale service #{
|
112
|
+
raise NetworkError.new "Could not scale #{service.type} #{service.name} due to error: #{e.message}"
|
117
113
|
end
|
118
114
|
end
|
119
115
|
|
@@ -6,12 +6,12 @@ module Scaltainer
|
|
6
6
|
|
7
7
|
def get_metrics(services)
|
8
8
|
services_count = services.keys.length rescue 0
|
9
|
-
raise Scaltainer::Warning.new "No
|
9
|
+
raise Scaltainer::Warning.new "No resources found for #{self.class.name}" if services_count == 0
|
10
10
|
end
|
11
11
|
|
12
12
|
def determine_desired_replicas(metric, service_config, current_replicas)
|
13
|
-
raise ConfigurationError.new 'No metric found for requested
|
14
|
-
raise ConfigurationError.new 'No configuration found for requested
|
13
|
+
raise ConfigurationError.new 'No metric found for requested resource' unless metric
|
14
|
+
raise ConfigurationError.new 'No configuration found for requested resource' unless service_config
|
15
15
|
end
|
16
16
|
|
17
17
|
def adjust_desired_replicas(desired_replicas, config)
|
@@ -32,7 +32,7 @@ module Scaltainer
|
|
32
32
|
yield
|
33
33
|
state["upscale_sensitivity"] = 0
|
34
34
|
else
|
35
|
-
logger.debug "Scaling up of
|
35
|
+
logger.debug "Scaling up of resource #{service_name} blocked by upscale_sensitivity at level " +
|
36
36
|
"#{state["upscale_sensitivity"]} while level #{config["upscale_sensitivity"]} is required"
|
37
37
|
end
|
38
38
|
elsif replica_diff < 0 # TODO force down when above max?
|
@@ -45,17 +45,17 @@ module Scaltainer
|
|
45
45
|
yield
|
46
46
|
state["downscale_sensitivity"] = 0
|
47
47
|
else
|
48
|
-
logger.debug "Scaling down of
|
48
|
+
logger.debug "Scaling down of resource #{service_name} blocked by downscale_sensitivity at level " +
|
49
49
|
"#{state["downscale_sensitivity"]} while level #{config["downscale_sensitivity"]} is required"
|
50
50
|
end
|
51
51
|
else
|
52
|
-
logger.debug "Scaling down of
|
52
|
+
logger.debug "Scaling down of resource #{service_name} to #{metric} replicas blocked by a non-decrementable config"
|
53
53
|
end
|
54
54
|
else
|
55
55
|
# no breach, change state
|
56
56
|
state["upscale_sensitivity"] = 0
|
57
57
|
state["downscale_sensitivity"] = 0
|
58
|
-
logger.info "No need to scale
|
58
|
+
logger.info "No need to scale resource #{service_name}"
|
59
59
|
end
|
60
60
|
end
|
61
61
|
|
@@ -14,7 +14,7 @@ module Scaltainer
|
|
14
14
|
|
15
15
|
services.reduce({}) do |hash, (service_name, service_config)|
|
16
16
|
app_id = service_config["newrelic_app_id"]
|
17
|
-
raise ConfigurationError.new "
|
17
|
+
raise ConfigurationError.new "Resource #{service_name} does not have a corresponding newrelic_app_id" unless app_id
|
18
18
|
|
19
19
|
begin
|
20
20
|
metric = nr.get_avg_response_time app_id, from, to
|
@@ -28,8 +28,8 @@ module Scaltainer
|
|
28
28
|
|
29
29
|
def determine_desired_replicas(metric, service_config, current_replicas)
|
30
30
|
super
|
31
|
-
raise ConfigurationError.new "Missing max_response_time in web
|
32
|
-
raise ConfigurationError.new "Missing min_response_time in web
|
31
|
+
raise ConfigurationError.new "Missing max_response_time in web resource configuration" unless service_config["max_response_time"]
|
32
|
+
raise ConfigurationError.new "Missing min_response_time in web resource configuration" unless service_config["min_response_time"]
|
33
33
|
unless service_config["min_response_time"] <= service_config["max_response_time"]
|
34
34
|
raise ConfigurationError.new "min_response_time and max_response_time are not in order"
|
35
35
|
end
|
@@ -21,7 +21,7 @@ module Scaltainer
|
|
21
21
|
|
22
22
|
def determine_desired_replicas(metric, service_config, current_replicas)
|
23
23
|
super
|
24
|
-
raise ConfigurationError.new "Missing ratio in worker
|
24
|
+
raise ConfigurationError.new "Missing ratio in worker resource configuration" unless service_config["ratio"]
|
25
25
|
if !metric.is_a?(Integer) || metric < 0
|
26
26
|
raise ConfigurationError.new "#{metric} is an invalid metric value, must be a non-negative number"
|
27
27
|
end
|
data/lib/scaltainer/version.rb
CHANGED
data/scaltainer.gemspec
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: scaltainer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Hossam Hammady
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-03-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -108,6 +108,20 @@ dependencies:
|
|
108
108
|
- - ">="
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: kubeclient
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - ">="
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - ">="
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: '0'
|
111
125
|
- !ruby/object:Gem::Dependency
|
112
126
|
name: dotenv
|
113
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -132,6 +146,7 @@ executables:
|
|
132
146
|
extensions: []
|
133
147
|
extra_rdoc_files: []
|
134
148
|
files:
|
149
|
+
- ".dockerignore"
|
135
150
|
- ".gitignore"
|
136
151
|
- ".rspec"
|
137
152
|
- ".travis.yml"
|
@@ -145,9 +160,12 @@ files:
|
|
145
160
|
- exe/scaltainer
|
146
161
|
- lib/scaltainer.rb
|
147
162
|
- lib/scaltainer/command.rb
|
148
|
-
- lib/scaltainer/docker/service.rb
|
149
163
|
- lib/scaltainer/exceptions.rb
|
150
164
|
- lib/scaltainer/newrelic/metrics.rb
|
165
|
+
- lib/scaltainer/orchestrators.rb
|
166
|
+
- lib/scaltainer/orchestrators/base.rb
|
167
|
+
- lib/scaltainer/orchestrators/kubernetes.rb
|
168
|
+
- lib/scaltainer/orchestrators/swarm.rb
|
151
169
|
- lib/scaltainer/runner.rb
|
152
170
|
- lib/scaltainer/service_types.rb
|
153
171
|
- lib/scaltainer/service_types/base.rb
|