reschedule 0.0.1 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/Dockerfile.example +28 -0
- data/README.md +92 -2
- data/bin/reschedule +11 -0
- data/docs/logo.png +0 -0
- data/lib/reschedule.rb +24 -1
- data/lib/reschedule/configuration.rb +16 -0
- data/lib/reschedule/heapster/api.rb +25 -0
- data/lib/reschedule/kubernetes/api.rb +36 -0
- data/lib/reschedule/manager.rb +22 -0
- data/lib/reschedule/reschedulers/all.rb +37 -0
- data/lib/reschedule/reschedulers/base.rb +38 -0
- data/lib/reschedule/reschedulers/memory_threshold.rb +83 -0
- data/lib/reschedule/version.rb +1 -1
- data/reschedule.gemspec +10 -3
- data/reschedule.yml.example +10 -0
- data/spec/lib/reschedule/reschedulers/all_spec.rb +45 -0
- data/spec/lib/reschedule/reschedulers/memory_threshold_spec.rb +53 -0
- data/spec/spec_helper.rb +4 -0
- data/spec/support/reschedule_spec_helper.rb +106 -0
- metadata +107 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d1607c6ac36e5e386b42184f637c468a8b33998e
|
4
|
+
data.tar.gz: a32ac6318bc36963ac2e55c5fff4186cc8dd5a8f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0bb2127120499219b253925b29b08d7d64cfcc86fb55034614acc4110b985b346bb00e10f94f90117f2ad8bf5bd8b39bf97a7f512a088b7500682194f982c300
|
7
|
+
data.tar.gz: 135d551ad50d2168114a4fd2fb67b54c4d611599374420bacc35d688f80b84a94dd5a78fcea1cadc32468f12400e83f9cf95c41498bc48e869da96476be79f59
|
data/.gitignore
CHANGED
data/Dockerfile.example
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
FROM ruby:2.2.2
|
2
|
+
|
3
|
+
RUN apt-get update -qq && apt-get install -y build-essential
|
4
|
+
|
5
|
+
ENV APP_HOME /app
|
6
|
+
RUN mkdir $APP_HOME
|
7
|
+
WORKDIR $APP_HOME
|
8
|
+
|
9
|
+
ADD Gemfile* $APP_HOME/
|
10
|
+
ADD reschedule.gemspec $APP_HOME/
|
11
|
+
ADD lib/reschedule/version.rb $APP_HOME/lib/reschedule/version.rb
|
12
|
+
RUN bundle install
|
13
|
+
|
14
|
+
ADD . $APP_HOME
|
15
|
+
|
16
|
+
ENV KUBERNETES_API_URL https://kubernetes.mycompany.com/api/
|
17
|
+
|
18
|
+
# Authentication option 1: HTTP basic auth
|
19
|
+
ENV KUBERNETES_API_USERNAME myusername
|
20
|
+
ENV KUBERNETES_API_PASSWORD mypassword
|
21
|
+
|
22
|
+
# Authentication option 2: client certificate
|
23
|
+
ENV KUBERNETES_API_CLIENT_KEY myclientkey
|
24
|
+
ENV KUBERNETES_API_CLIENT_CERT myclientcert
|
25
|
+
ENV KUBERNETES_API_CA_FILE path/to/my/ca/file
|
26
|
+
|
27
|
+
ENTRYPOINT ["bundle", "exec"]
|
28
|
+
CMD ["./bin/reschedule"]
|
data/README.md
CHANGED
@@ -1,9 +1,99 @@
|
|
1
|
-
Reschedule
|
2
|
-
|
1
|
+
<h1><img src="docs/logo.png?raw=true" width="27" /> Reschedule</h1>
|
2
|
+
|
3
|
+
Automatic, configurable Kubernetes rescheduling
|
3
4
|
|
4
5
|
Overview
|
5
6
|
--------
|
6
7
|
|
8
|
+
Reschedule lets you configure periodic and/or conditions-based rescheduling of Kubernetes pods. For example, you could configure Reschedule to:
|
9
|
+
|
10
|
+
* Check the memory usage of each node every 30 minutes. If the memory usage of a node is above 80%, reschedule the node's pods to more evenly distribute memory usage across the cluster.
|
11
|
+
* Restart all of a specific replication controller's pods every 24 hours as a stopgap for slow memory leaks.
|
12
|
+
|
13
|
+
Usage
|
14
|
+
-----
|
15
|
+
|
16
|
+
Reschedule runs as a pod on your Kubernetes cluster:
|
17
|
+
|
18
|
+
1. Clone this repo
|
19
|
+
1. Create `Dockerfile` and `reschedule.yml` using the `.example` files as references (see [Configuration](#configuration))
|
20
|
+
1. Build and run the Docker image on your Kubernetes cluster
|
21
|
+
|
22
|
+
Configuration
|
23
|
+
-------------
|
24
|
+
|
25
|
+
### reschedule.yml
|
26
|
+
|
27
|
+
A rescheduler is a periodic task that reschedules pods based on conditions. You can define as many reschedulers as you like in `reschedule.yml`.
|
28
|
+
|
29
|
+
For example, the following `reschedule.yml` will:
|
30
|
+
* Every 12 hours, reschedule all pods of replication controllers with names matching the regex `/aggregation\-(fast|slow)/`
|
31
|
+
* Every 30 minutes, reschedule all pods in the namespace `mynamespace` on nodes with >80% memory usage
|
32
|
+
|
33
|
+
```yaml
|
34
|
+
reschedulers:
|
35
|
+
-
|
36
|
+
type: All
|
37
|
+
every: 12h
|
38
|
+
options:
|
39
|
+
replication_controller_name_match: aggregation\-(fast|slow)
|
40
|
+
-
|
41
|
+
type: MemoryThreshold
|
42
|
+
every: 30m
|
43
|
+
options:
|
44
|
+
memory_threshold: 0.8
|
45
|
+
namespace: mynamespace
|
46
|
+
```
|
47
|
+
|
48
|
+
### Rescheduler types
|
49
|
+
|
50
|
+
#### All
|
51
|
+
|
52
|
+
Reschedules all pods.
|
53
|
+
|
54
|
+
Options:
|
55
|
+
|
56
|
+
* `namespace` - The namespace of the pods
|
57
|
+
* `replication_controller_name_match` - Only pods of replication controllers with names that match this regex will be rescheduled.
|
58
|
+
|
59
|
+
#### MemoryThreshold
|
60
|
+
|
61
|
+
Reschedules all pods on nodes with memory usage greater than the threshold.
|
62
|
+
|
63
|
+
Options:
|
64
|
+
|
65
|
+
* `namespace` - The namespace of the pods
|
66
|
+
* `memory_threshold` - The memory threshold (0.8 == 80%)
|
67
|
+
|
68
|
+
### Dockerfile
|
69
|
+
|
70
|
+
See `Dockerfile.example` for an example.
|
71
|
+
|
72
|
+
#### Authentication
|
73
|
+
|
74
|
+
To authenticate to the Kubernetes API, you have a couple of options:
|
75
|
+
|
76
|
+
```bash
|
77
|
+
# Authentication option 1: HTTP basic auth
|
78
|
+
ENV KUBERNETES_API_USERNAME myusername
|
79
|
+
ENV KUBERNETES_API_PASSWORD mypassword
|
80
|
+
|
81
|
+
# Authentication option 2: client certificate
|
82
|
+
ENV KUBERNETES_API_CLIENT_KEY myclientkey
|
83
|
+
ENV KUBERNETES_API_CLIENT_CERT myclientcert
|
84
|
+
ENV KUBERNETES_API_CA_FILE path/to/my/ca/file
|
85
|
+
```
|
86
|
+
|
87
|
+
#### Dry run
|
88
|
+
|
89
|
+
You can put Reschedule in dry run mode with:
|
90
|
+
|
91
|
+
```bash
|
92
|
+
ENV RESCHEDULE_DRY_RUN 1
|
93
|
+
```
|
94
|
+
|
95
|
+
In this mode, Reschedule won't perform any rescheduling, but it will log the rescheduling that it would do.
|
96
|
+
|
7
97
|
License
|
8
98
|
-------
|
9
99
|
|
data/bin/reschedule
ADDED
data/docs/logo.png
ADDED
Binary file
|
data/lib/reschedule.rb
CHANGED
@@ -1,5 +1,28 @@
|
|
1
|
+
require 'active_support/all'
|
2
|
+
require 'httparty'
|
3
|
+
require 'kubeclient'
|
4
|
+
require 'rufus-scheduler'
|
5
|
+
|
1
6
|
directory = File.dirname(File.absolute_path(__FILE__))
|
2
|
-
Dir.glob("#{directory}/reschedule
|
7
|
+
Dir.glob("#{directory}/reschedule/reschedulers/base.rb") { |file| require file }
|
8
|
+
Dir.glob("#{directory}/reschedule/**/*.rb") { |file| require file }
|
3
9
|
|
4
10
|
module Reschedule
|
11
|
+
module_function
|
12
|
+
|
13
|
+
def self.configure
|
14
|
+
yield(configuration) if block_given?
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.configuration
|
18
|
+
@configuration ||= Configuration.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.logger
|
22
|
+
@logger ||= begin
|
23
|
+
logger = Logger.new(STDOUT)
|
24
|
+
logger.progname = 'Reschedule'
|
25
|
+
logger
|
26
|
+
end
|
27
|
+
end
|
5
28
|
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Reschedule
|
2
|
+
class Configuration
|
3
|
+
attr_accessor :dry_run, :kubernetes_api_url, :kubernetes_api_username, :kubernetes_api_password,
|
4
|
+
:kubernetes_api_client_key, :kubernetes_api_client_cert, :kubernetes_api_ca_file
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
self.dry_run = ENV['RESCHEDULE_DRY_RUN'] == '1'
|
8
|
+
self.kubernetes_api_url = ENV['KUBERNETES_API_URL']
|
9
|
+
self.kubernetes_api_username = ENV['KUBERNETES_API_USERNAME']
|
10
|
+
self.kubernetes_api_password = ENV['KUBERNETES_API_PASSWORD']
|
11
|
+
self.kubernetes_api_client_key = ENV['KUBERNETES_API_CLIENT_KEY']
|
12
|
+
self.kubernetes_api_client_cert = ENV['KUBERNETES_API_CLIENT_CERT']
|
13
|
+
self.kubernetes_api_ca_file = ENV['KUBERNETES_API_CA_FILE']
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Reschedule
|
2
|
+
module Heapster
|
3
|
+
class Api
|
4
|
+
def get_node_stats(node_name)
|
5
|
+
get("model/nodes/#{node_name}/stats/")
|
6
|
+
end
|
7
|
+
|
8
|
+
def get_nodes
|
9
|
+
get('model/nodes/')
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def get(path)
|
15
|
+
auth = {
|
16
|
+
username: Reschedule.configuration.kubernetes_api_username,
|
17
|
+
password: Reschedule.configuration.kubernetes_api_password
|
18
|
+
}
|
19
|
+
base_url = "#{Reschedule.configuration.kubernetes_api_url}v1/proxy/namespaces/kube-system/services/heapster/api/v1/"
|
20
|
+
url = "#{base_url}#{path}"
|
21
|
+
HTTParty.get(url, basic_auth: auth, verify: false)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module Reschedule
|
2
|
+
module Kubernetes
|
3
|
+
class Api
|
4
|
+
attr_accessor :client
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
self.client = Kubeclient::Client.new(Reschedule.configuration.kubernetes_api_url, 'v1', client_options)
|
8
|
+
end
|
9
|
+
|
10
|
+
def method_missing(method, *args)
|
11
|
+
self.client.public_send(method, *args)
|
12
|
+
end
|
13
|
+
|
14
|
+
private
|
15
|
+
|
16
|
+
def client_options
|
17
|
+
client_options = {}
|
18
|
+
if Reschedule.configuration.kubernetes_api_username.present?
|
19
|
+
client_options[:auth_options] = {
|
20
|
+
user: Reschedule.configuration.kubernetes_api_username,
|
21
|
+
password: Reschedule.configuration.kubernetes_api_password
|
22
|
+
}
|
23
|
+
end
|
24
|
+
if Reschedule.configuration.kubernetes_api_client_key.present?
|
25
|
+
client_options[:ssl_options] = {
|
26
|
+
client_key: Reschedule.configuration.kubernetes_api_client_key,
|
27
|
+
client_cert: Reschedule.configuration.kubernetes_api_client_cert,
|
28
|
+
ca_file: Reschedule.configuration.kubernetes_api_ca_file,
|
29
|
+
verify_ssl: OpenSSL::SSL::VERIFY_PEER
|
30
|
+
}
|
31
|
+
end
|
32
|
+
client_options
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Reschedule
|
2
|
+
class Manager
|
3
|
+
def run
|
4
|
+
scheduler = Rufus::Scheduler.new
|
5
|
+
|
6
|
+
config['reschedulers'].each do |rescheduler|
|
7
|
+
klass = "Reschedule::Reschedulers::#{rescheduler['type']}".constantize
|
8
|
+
scheduler.every rescheduler['every'] do
|
9
|
+
klass.new(rescheduler['options']).run
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
scheduler.join
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def config
|
19
|
+
@config ||= YAML.load(File.read('./reschedule.yml'))
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Reschedule
|
2
|
+
module Reschedulers
|
3
|
+
class All < Base
|
4
|
+
attr_reader :kubernetes_api, :replication_controller_name_regex
|
5
|
+
|
6
|
+
self.defaults = {
|
7
|
+
'namespace' => 'default'
|
8
|
+
}
|
9
|
+
|
10
|
+
def initialize(options={})
|
11
|
+
super
|
12
|
+
@kubernetes_api = Reschedule::Kubernetes::Api.new
|
13
|
+
@replication_controller_name_regex = options['replication_controller_name_match'].present? ? Regexp.new(options['replication_controller_name_match']) : nil
|
14
|
+
end
|
15
|
+
|
16
|
+
def run
|
17
|
+
super
|
18
|
+
replication_controllers = kubernetes_api.get_replication_controllers(namespace: options['namespace'])
|
19
|
+
Reschedule.logger.debug "Found #{replication_controllers.length} replication controllers"
|
20
|
+
|
21
|
+
replication_controllers.select! do |replication_controller|
|
22
|
+
template = replication_controller.spec[:template] || replication_controller.spec['table'][:template]
|
23
|
+
image = template['spec']['containers'][0]['image']
|
24
|
+
next if image.include?('/reschedule')
|
25
|
+
next if replication_controller.spec.replicas == 0
|
26
|
+
next if replication_controller_name_regex && replication_controller.metadata.name !~ replication_controller_name_regex
|
27
|
+
true
|
28
|
+
end
|
29
|
+
|
30
|
+
Reschedule.logger.debug "Found #{replication_controllers.length} replication controllers to reschedule"
|
31
|
+
replication_controllers.each do |replication_controller|
|
32
|
+
reschedule_replication_controller(replication_controller.metadata.name)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Reschedule
|
2
|
+
module Reschedulers
|
3
|
+
class Base
|
4
|
+
attr_reader :options
|
5
|
+
|
6
|
+
class_attribute :defaults
|
7
|
+
|
8
|
+
def initialize(options={})
|
9
|
+
@options = options || {}
|
10
|
+
@options.reverse_merge!(self.class.defaults)
|
11
|
+
end
|
12
|
+
|
13
|
+
def run
|
14
|
+
Reschedule.logger.debug "Starting #{self.class.name}"
|
15
|
+
Reschedule.logger.debug "Dry run mode is on" if Reschedule.configuration.dry_run
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def reschedule_replication_controller(replication_controller_name)
|
21
|
+
Reschedule.logger.debug "Rescheduling #{replication_controller_name}"
|
22
|
+
|
23
|
+
unless Reschedule.configuration.dry_run
|
24
|
+
replication_controller = kubernetes_api.get_replication_controllers(label_selector: "name=#{replication_controller_name}").first
|
25
|
+
original_replicas = replication_controller.spec.replicas
|
26
|
+
replication_controller.spec.replicas = 0
|
27
|
+
kubernetes_api.update_replication_controller(replication_controller)
|
28
|
+
sleep 0.5
|
29
|
+
replication_controller = kubernetes_api.get_replication_controllers(label_selector: "name=#{replication_controller_name}").first
|
30
|
+
replication_controller.spec.replicas = original_replicas
|
31
|
+
kubernetes_api.update_replication_controller(replication_controller)
|
32
|
+
end
|
33
|
+
|
34
|
+
Reschedule.logger.debug "Rescheduled #{replication_controller_name}"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
module Reschedule
|
2
|
+
module Reschedulers
|
3
|
+
class MemoryThreshold < Base
|
4
|
+
attr_reader :heapster_api, :kubernetes_api
|
5
|
+
|
6
|
+
self.defaults = {
|
7
|
+
'memory_threshold' => 0.8,
|
8
|
+
'namespace' => 'default'
|
9
|
+
}
|
10
|
+
|
11
|
+
def initialize(options={})
|
12
|
+
super
|
13
|
+
@heapster_api = Reschedule::Heapster::Api.new
|
14
|
+
@kubernetes_api = Reschedule::Kubernetes::Api.new
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
super
|
19
|
+
node_names = rescheduleable_node_names
|
20
|
+
Reschedule.logger.debug "Found #{node_names.length} rescheduleable nodes"
|
21
|
+
return if node_names.blank?
|
22
|
+
|
23
|
+
pods = get_pods
|
24
|
+
Reschedule.logger.debug "Found #{pods.length} total pods"
|
25
|
+
replication_controller_names = pods_and_node_names_to_replication_controller_names(pods, node_names)
|
26
|
+
replication_controller_names.each do |replication_controller_name|
|
27
|
+
reschedule_replication_controller(replication_controller_name)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def get_pods
|
34
|
+
pods = kubernetes_api.get_pods
|
35
|
+
pods.select { |pod| pod.metadata.namespace == options['namespace'] }
|
36
|
+
end
|
37
|
+
|
38
|
+
def pods_and_node_names_to_replication_controller_names(pods, node_names)
|
39
|
+
node_pods = pods.select do |pod|
|
40
|
+
node_names.include?(pod.spec.nodeName)
|
41
|
+
end
|
42
|
+
pods_to_replication_controller_names(node_pods)
|
43
|
+
end
|
44
|
+
|
45
|
+
def pods_to_replication_controller_names(pods)
|
46
|
+
replication_controller_names = pods.map do |pod|
|
47
|
+
created_by = pod.metadata.annotations['kubernetes.io/created-by']
|
48
|
+
created_by = JSON.parse(created_by)
|
49
|
+
created_by['reference']['name']
|
50
|
+
end
|
51
|
+
replication_controller_names.uniq
|
52
|
+
end
|
53
|
+
|
54
|
+
def rescheduleable_node_names
|
55
|
+
nodes = heapster_api.get_nodes
|
56
|
+
if nodes.code != 200
|
57
|
+
Reschedule.logger.debug "Unable to get nodes in Heapster: #{nodes}"
|
58
|
+
return
|
59
|
+
end
|
60
|
+
Reschedule.logger.debug "Found #{nodes.length} nodes in Heapster"
|
61
|
+
nodes_to_rescheduleable_node_names(nodes)
|
62
|
+
end
|
63
|
+
|
64
|
+
def nodes_to_rescheduleable_node_names(nodes)
|
65
|
+
rescheduleable_node_names = []
|
66
|
+
nodes.each do |node|
|
67
|
+
node_name = node['name']
|
68
|
+
node_stats = heapster_api.get_node_stats(node_name)
|
69
|
+
if node_stats.code != 200
|
70
|
+
Reschedule.logger.debug "Node stats not found: #{node_stats}"
|
71
|
+
next
|
72
|
+
end
|
73
|
+
memory_limit = node_stats['stats']['memory-limit']['hour']['average']
|
74
|
+
memory_usage = node_stats['stats']['memory-usage']['hour']['average']
|
75
|
+
if memory_usage.to_f / memory_limit > options['memory_threshold']
|
76
|
+
rescheduleable_node_names << node_name
|
77
|
+
end
|
78
|
+
end
|
79
|
+
rescheduleable_node_names
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
data/lib/reschedule/version.rb
CHANGED
data/reschedule.gemspec
CHANGED
@@ -3,8 +3,8 @@ require File.expand_path('../lib/reschedule/version', __FILE__)
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.authors = ['Tom Benner']
|
5
5
|
s.email = ['tombenner@gmail.com']
|
6
|
-
s.description = s.summary = %q{
|
7
|
-
s.homepage = 'https://github.com/
|
6
|
+
s.description = s.summary = %q{Automatic, configurable Kubernetes rescheduling}
|
7
|
+
s.homepage = 'https://github.com/entelo/reschedule'
|
8
8
|
|
9
9
|
s.files = `git ls-files`.split($\)
|
10
10
|
s.name = 'reschedule'
|
@@ -12,5 +12,12 @@ Gem::Specification.new do |s|
|
|
12
12
|
s.version = Reschedule::VERSION
|
13
13
|
s.license = 'MIT'
|
14
14
|
|
15
|
-
s.
|
15
|
+
s.add_dependency 'activesupport', '~> 4.2'
|
16
|
+
s.add_dependency 'httparty', '~> 0.13'
|
17
|
+
s.add_dependency 'kubeclient', '~> 0.9.0'
|
18
|
+
s.add_dependency 'rufus-scheduler', '~> 3.2'
|
19
|
+
|
20
|
+
s.add_development_dependency 'hashie', '~> 3.4'
|
21
|
+
s.add_development_dependency 'rspec', '~> 3.0'
|
22
|
+
s.add_development_dependency 'webmock', '~> 1.8'
|
16
23
|
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Reschedule::Reschedulers::All do
|
4
|
+
include RescheduleSpecHelper
|
5
|
+
|
6
|
+
describe '#run' do
|
7
|
+
let(:reschedule_replication_controller) { create_replication_controller('foo.bar.com/reschedule:latest') }
|
8
|
+
let(:other_replication_controller) { create_replication_controller('foo.bar.com/other:latest') }
|
9
|
+
|
10
|
+
before :each do
|
11
|
+
allow_any_instance_of(Reschedule::Kubernetes::Api).to receive(:get_replication_controllers).and_return(replication_controllers)
|
12
|
+
allow_any_instance_of(Reschedule::Kubernetes::Api).to receive(:update_replication_controller)
|
13
|
+
allow_any_instance_of(Reschedule::Kubernetes::Api).to receive(:get_pods).and_return(pods)
|
14
|
+
end
|
15
|
+
|
16
|
+
context 'default' do
|
17
|
+
context 'other replication controller' do
|
18
|
+
let(:replication_controllers) { [other_replication_controller] }
|
19
|
+
|
20
|
+
it 'updates all of the replication controllers' do
|
21
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to receive(:update_replication_controller).with(replication_controllers[0])
|
22
|
+
described_class.new.run
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
context 'reschedule replication controller' do
|
27
|
+
let(:replication_controllers) { [reschedule_replication_controller] }
|
28
|
+
|
29
|
+
it 'does not update any replication controller' do
|
30
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to_not receive(:update_replication_controller)
|
31
|
+
described_class.new.run
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
context 'dry run' do
|
37
|
+
let(:dry_run) { true }
|
38
|
+
|
39
|
+
it 'does not update any replication controllers' do
|
40
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to_not receive(:update_replication_controller)
|
41
|
+
described_class.new.run
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Reschedule::Reschedulers::MemoryThreshold do
|
4
|
+
include RescheduleSpecHelper
|
5
|
+
|
6
|
+
describe '#run' do
|
7
|
+
before :each do
|
8
|
+
stub_json(:get, "#{heapster_url}model/nodes/", nodes)
|
9
|
+
stub_json(:get, "#{heapster_url}model/nodes/#{node_names[0]}/stats/", node_stats[0])
|
10
|
+
stub_json(:get, "#{heapster_url}model/nodes/#{node_names[1]}/stats/", node_stats[1])
|
11
|
+
|
12
|
+
allow_any_instance_of(Reschedule::Kubernetes::Api).to receive(:get_replication_controllers).with(label_selector: "name=#{replication_controller_names[0]}").and_return(replication_controllers)
|
13
|
+
allow_any_instance_of(Reschedule::Kubernetes::Api).to receive(:update_replication_controller)
|
14
|
+
allow_any_instance_of(Reschedule::Kubernetes::Api).to receive(:get_pods).and_return(pods)
|
15
|
+
end
|
16
|
+
|
17
|
+
context 'memory threshold not exceeded' do
|
18
|
+
it 'does not update any replication controllers' do
|
19
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to_not receive(:update_replication_controller)
|
20
|
+
described_class.new.run
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
context 'memory threshold exceeded' do
|
25
|
+
it 'updates the replication controller' do
|
26
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to receive(:update_replication_controller).with(replication_controllers[0])
|
27
|
+
described_class.new('memory_threshold' => 0.01).run
|
28
|
+
end
|
29
|
+
|
30
|
+
context 'dry run' do
|
31
|
+
let(:dry_run) { true }
|
32
|
+
|
33
|
+
it 'does not update any replication controllers' do
|
34
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to_not receive(:update_replication_controller)
|
35
|
+
described_class.new.run
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
context 'memory threshold exceeded in a non-default namespace' do
|
41
|
+
let(:pods) do
|
42
|
+
pod = pod_default
|
43
|
+
pod.metadata.namespace = 'foo'
|
44
|
+
[pod]
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'does not update any replication controllers' do
|
48
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to_not receive(:update_replication_controller)
|
49
|
+
described_class.new('memory_threshold' => 0.01).run
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -0,0 +1,106 @@
|
|
1
|
+
module RescheduleSpecHelper
|
2
|
+
extend RSpec::SharedContext
|
3
|
+
|
4
|
+
def stub_json(method, url, response_data)
|
5
|
+
response = {
|
6
|
+
status: 200,
|
7
|
+
body: response_data.to_json,
|
8
|
+
headers: { 'Content-Type' => 'application/json' }
|
9
|
+
}
|
10
|
+
stub_request(:get, url).to_return(response)
|
11
|
+
end
|
12
|
+
|
13
|
+
def create_replication_controller(image)
|
14
|
+
Hashie::Mash.new(
|
15
|
+
metadata: {
|
16
|
+
name: 'foo',
|
17
|
+
spec: {
|
18
|
+
replicas: 1
|
19
|
+
}
|
20
|
+
},
|
21
|
+
spec: {
|
22
|
+
table: {
|
23
|
+
template: {
|
24
|
+
spec: {
|
25
|
+
containers: [
|
26
|
+
{
|
27
|
+
image: image
|
28
|
+
}
|
29
|
+
]
|
30
|
+
}
|
31
|
+
}
|
32
|
+
}
|
33
|
+
}
|
34
|
+
)
|
35
|
+
end
|
36
|
+
|
37
|
+
before :each do
|
38
|
+
Reschedule.configure do |config|
|
39
|
+
config.kubernetes_api_url = "https://#{kube_host}/api/"
|
40
|
+
config.kubernetes_api_username = kube_username
|
41
|
+
config.kubernetes_api_password = kube_password
|
42
|
+
config.dry_run = dry_run
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
let(:kube_host) { 'kube.local' }
|
47
|
+
let(:kube_username) { 'foo' }
|
48
|
+
let(:kube_password) { 'bar' }
|
49
|
+
let(:kube_url) { "https://#{kube_username}:#{kube_password}@#{kube_host}/api/" }
|
50
|
+
let(:heapster_url) { "#{kube_url}v1/proxy/namespaces/kube-system/services/heapster/api/v1/" }
|
51
|
+
let(:dry_run) { false }
|
52
|
+
|
53
|
+
let(:node_names) { (0..1).map { |i| "node-#{i}.compute.internal" } }
|
54
|
+
let(:replication_controller_names) { (0..1).map { |i| "rc-#{i}" } }
|
55
|
+
|
56
|
+
let(:node_stats) { [node_stat_default, node_stat_default] }
|
57
|
+
let(:nodes) do
|
58
|
+
(0..1).map do |i|
|
59
|
+
{
|
60
|
+
'name' => node_names[i],
|
61
|
+
'cpuUsage' => 306,
|
62
|
+
'memUsage' => 2831523840
|
63
|
+
}
|
64
|
+
end
|
65
|
+
end
|
66
|
+
let(:pods) { [pod_default] }
|
67
|
+
let(:replication_controllers) { [double.as_null_object] }
|
68
|
+
|
69
|
+
let(:node_stat_default) do
|
70
|
+
{
|
71
|
+
"uptime"=>401576,
|
72
|
+
"stats"=>
|
73
|
+
{"cpu-limit"=>
|
74
|
+
{"minute"=>{"average"=>36000, "percentile"=>36000, "max"=>36000},
|
75
|
+
"hour"=>{"average"=>36000, "percentile"=>36000, "max"=>36000},
|
76
|
+
"day"=>{"average"=>36000, "percentile"=>36000, "max"=>36000}},
|
77
|
+
"cpu-usage"=>
|
78
|
+
{"minute"=>{"average"=>306, "percentile"=>306, "max"=>306}, "hour"=>{"average"=>317, "percentile"=>340, "max"=>350}, "day"=>{"average"=>316, "percentile"=>330, "max"=>330}},
|
79
|
+
"memory-limit"=>
|
80
|
+
{"minute"=>{"average"=>63317110784, "percentile"=>63317110784, "max"=>63317110784},
|
81
|
+
"hour"=>{"average"=>63317213184, "percentile"=>63317213184, "max"=>63317213184},
|
82
|
+
"day"=>{"average"=>63317213184, "percentile"=>63317213184, "max"=>63317213184}},
|
83
|
+
"memory-usage"=>
|
84
|
+
{"minute"=>{"average"=>4992581632, "percentile"=>4992581632, "max"=>4992581632},
|
85
|
+
"hour"=>{"average"=>4982343816, "percentile"=>4991221760, "max"=>4995416064},
|
86
|
+
"day"=>{"average"=>4968991948, "percentile"=>4978638848, "max"=>4979036160}},
|
87
|
+
"memory-working"=>
|
88
|
+
{"minute"=>{"average"=>2815111168, "percentile"=>2815111168, "max"=>2815111168},
|
89
|
+
"hour"=>{"average"=>2810043869, "percentile"=>2814377984, "max"=>2818572288},
|
90
|
+
"day"=>{"average"=>2805639850, "percentile"=>2810183680, "max"=>2810183680}}}}
|
91
|
+
end
|
92
|
+
|
93
|
+
let(:pod_default) do
|
94
|
+
Hashie::Mash.new({
|
95
|
+
spec: { nodeName: node_names[0] },
|
96
|
+
metadata: {
|
97
|
+
annotations: {
|
98
|
+
'kubernetes.io/created-by' => {
|
99
|
+
reference: { name: replication_controller_names[0] }
|
100
|
+
}.to_json
|
101
|
+
},
|
102
|
+
namespace: 'default'
|
103
|
+
}
|
104
|
+
})
|
105
|
+
end
|
106
|
+
end
|
metadata
CHANGED
@@ -1,30 +1,114 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: reschedule
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tom Benner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-05-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '4.2'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '4.2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: httparty
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0.13'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0.13'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: kubeclient
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.9.0
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.9.0
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rufus-scheduler
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.2'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '3.2'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: hashie
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '3.4'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '3.4'
|
13
83
|
- !ruby/object:Gem::Dependency
|
14
84
|
name: rspec
|
15
85
|
requirement: !ruby/object:Gem::Requirement
|
16
86
|
requirements:
|
17
|
-
- - "
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '3.0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '3.0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: webmock
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
18
102
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
103
|
+
version: '1.8'
|
20
104
|
type: :development
|
21
105
|
prerelease: false
|
22
106
|
version_requirements: !ruby/object:Gem::Requirement
|
23
107
|
requirements:
|
24
|
-
- - "
|
108
|
+
- - "~>"
|
25
109
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
27
|
-
description:
|
110
|
+
version: '1.8'
|
111
|
+
description: Automatic, configurable Kubernetes rescheduling
|
28
112
|
email:
|
29
113
|
- tombenner@gmail.com
|
30
114
|
executables: []
|
@@ -32,15 +116,29 @@ extensions: []
|
|
32
116
|
extra_rdoc_files: []
|
33
117
|
files:
|
34
118
|
- ".gitignore"
|
119
|
+
- Dockerfile.example
|
35
120
|
- Gemfile
|
36
121
|
- MIT-LICENSE
|
37
122
|
- README.md
|
38
123
|
- Rakefile
|
124
|
+
- bin/reschedule
|
125
|
+
- docs/logo.png
|
39
126
|
- lib/reschedule.rb
|
127
|
+
- lib/reschedule/configuration.rb
|
128
|
+
- lib/reschedule/heapster/api.rb
|
129
|
+
- lib/reschedule/kubernetes/api.rb
|
130
|
+
- lib/reschedule/manager.rb
|
131
|
+
- lib/reschedule/reschedulers/all.rb
|
132
|
+
- lib/reschedule/reschedulers/base.rb
|
133
|
+
- lib/reschedule/reschedulers/memory_threshold.rb
|
40
134
|
- lib/reschedule/version.rb
|
41
135
|
- reschedule.gemspec
|
136
|
+
- reschedule.yml.example
|
137
|
+
- spec/lib/reschedule/reschedulers/all_spec.rb
|
138
|
+
- spec/lib/reschedule/reschedulers/memory_threshold_spec.rb
|
42
139
|
- spec/spec_helper.rb
|
43
|
-
|
140
|
+
- spec/support/reschedule_spec_helper.rb
|
141
|
+
homepage: https://github.com/entelo/reschedule
|
44
142
|
licenses:
|
45
143
|
- MIT
|
46
144
|
metadata: {}
|
@@ -63,5 +161,5 @@ rubyforge_project:
|
|
63
161
|
rubygems_version: 2.4.8
|
64
162
|
signing_key:
|
65
163
|
specification_version: 4
|
66
|
-
summary:
|
164
|
+
summary: Automatic, configurable Kubernetes rescheduling
|
67
165
|
test_files: []
|