reschedule 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/Dockerfile.example +28 -0
- data/README.md +92 -2
- data/bin/reschedule +11 -0
- data/docs/logo.png +0 -0
- data/lib/reschedule.rb +24 -1
- data/lib/reschedule/configuration.rb +16 -0
- data/lib/reschedule/heapster/api.rb +25 -0
- data/lib/reschedule/kubernetes/api.rb +36 -0
- data/lib/reschedule/manager.rb +22 -0
- data/lib/reschedule/reschedulers/all.rb +37 -0
- data/lib/reschedule/reschedulers/base.rb +38 -0
- data/lib/reschedule/reschedulers/memory_threshold.rb +83 -0
- data/lib/reschedule/version.rb +1 -1
- data/reschedule.gemspec +10 -3
- data/reschedule.yml.example +10 -0
- data/spec/lib/reschedule/reschedulers/all_spec.rb +45 -0
- data/spec/lib/reschedule/reschedulers/memory_threshold_spec.rb +53 -0
- data/spec/spec_helper.rb +4 -0
- data/spec/support/reschedule_spec_helper.rb +106 -0
- metadata +107 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d1607c6ac36e5e386b42184f637c468a8b33998e
|
4
|
+
data.tar.gz: a32ac6318bc36963ac2e55c5fff4186cc8dd5a8f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0bb2127120499219b253925b29b08d7d64cfcc86fb55034614acc4110b985b346bb00e10f94f90117f2ad8bf5bd8b39bf97a7f512a088b7500682194f982c300
|
7
|
+
data.tar.gz: 135d551ad50d2168114a4fd2fb67b54c4d611599374420bacc35d688f80b84a94dd5a78fcea1cadc32468f12400e83f9cf95c41498bc48e869da96476be79f59
|
data/.gitignore
CHANGED
data/Dockerfile.example
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
FROM ruby:2.2.2
|
2
|
+
|
3
|
+
RUN apt-get update -qq && apt-get install -y build-essential
|
4
|
+
|
5
|
+
ENV APP_HOME /app
|
6
|
+
RUN mkdir $APP_HOME
|
7
|
+
WORKDIR $APP_HOME
|
8
|
+
|
9
|
+
ADD Gemfile* $APP_HOME/
|
10
|
+
ADD reschedule.gemspec $APP_HOME/
|
11
|
+
ADD lib/reschedule/version.rb $APP_HOME/lib/reschedule/version.rb
|
12
|
+
RUN bundle install
|
13
|
+
|
14
|
+
ADD . $APP_HOME
|
15
|
+
|
16
|
+
ENV KUBERNETES_API_URL https://kubernetes.mycompany.com/api/
|
17
|
+
|
18
|
+
# Authentication option 1: HTTP basic auth
|
19
|
+
ENV KUBERNETES_API_USERNAME myusername
|
20
|
+
ENV KUBERNETES_API_PASSWORD mypassword
|
21
|
+
|
22
|
+
# Authentication option 2: client certificate
|
23
|
+
ENV KUBERNETES_API_CLIENT_KEY myclientkey
|
24
|
+
ENV KUBERNETES_API_CLIENT_CERT myclientcert
|
25
|
+
ENV KUBERNETES_API_CA_FILE path/to/my/ca/file
|
26
|
+
|
27
|
+
ENTRYPOINT ["bundle", "exec"]
|
28
|
+
CMD ["./bin/reschedule"]
|
data/README.md
CHANGED
@@ -1,9 +1,99 @@
|
|
1
|
-
Reschedule
|
2
|
-
|
1
|
+
<h1><img src="docs/logo.png?raw=true" width="27" /> Reschedule</h1>
|
2
|
+
|
3
|
+
Automatic, configurable Kubernetes rescheduling
|
3
4
|
|
4
5
|
Overview
|
5
6
|
--------
|
6
7
|
|
8
|
+
Reschedule lets you configure periodic and/or conditions-based rescheduling of Kubernetes pods. For example, you could configure Reschedule to:
|
9
|
+
|
10
|
+
* Check the memory usage of each node every 30 minutes. If the memory usage of a node is above 80%, reschedule the node's pods to more evenly distribute memory usage across the cluster.
|
11
|
+
* Restart all of a specific replication controller's pods every 24 hours as a stopgap for slow memory leaks.
|
12
|
+
|
13
|
+
Usage
|
14
|
+
-----
|
15
|
+
|
16
|
+
Reschedule runs as a pod on your Kubernetes cluster:
|
17
|
+
|
18
|
+
1. Clone this repo
|
19
|
+
1. Create `Dockerfile` and `reschedule.yml` using the `.example` files as references (see [Configuration](#configuration))
|
20
|
+
1. Build and run the Docker image on your Kubernetes cluster
|
21
|
+
|
22
|
+
Configuration
|
23
|
+
-------------
|
24
|
+
|
25
|
+
### reschedule.yml
|
26
|
+
|
27
|
+
A rescheduler is a periodic task that reschedules pods based on conditions. You can define as many reschedulers as you like in `reschedule.yml`.
|
28
|
+
|
29
|
+
For example, the following `reschedule.yml` will:
|
30
|
+
* Every 12 hours, reschedule all pods of replication controllers with names matching the regex `/aggregation\-(fast|slow)/`
|
31
|
+
* Every 30 minutes, reschedule all pods in the namespace `mynamespace` on nodes with >80% memory usage
|
32
|
+
|
33
|
+
```yaml
|
34
|
+
reschedulers:
|
35
|
+
-
|
36
|
+
type: All
|
37
|
+
every: 12h
|
38
|
+
options:
|
39
|
+
replication_controller_name_match: aggregation\-(fast|slow)
|
40
|
+
-
|
41
|
+
type: MemoryThreshold
|
42
|
+
every: 30m
|
43
|
+
options:
|
44
|
+
memory_threshold: 0.8
|
45
|
+
namespace: mynamespace
|
46
|
+
```
|
47
|
+
|
48
|
+
### Rescheduler types
|
49
|
+
|
50
|
+
#### All
|
51
|
+
|
52
|
+
Reschedules all pods.
|
53
|
+
|
54
|
+
Options:
|
55
|
+
|
56
|
+
* `namespace` - The namespace of the pods
|
57
|
+
* `replication_controller_name_match` - Only pods of replication controllers with names that match this regex will be rescheduled.
|
58
|
+
|
59
|
+
#### MemoryThreshold
|
60
|
+
|
61
|
+
Reschedules all pods on nodes with memory usage greater than the threshold.
|
62
|
+
|
63
|
+
Options:
|
64
|
+
|
65
|
+
* `namespace` - The namespace of the pods
|
66
|
+
* `memory_threshold` - The memory threshold (0.8 == 80%)
|
67
|
+
|
68
|
+
### Dockerfile
|
69
|
+
|
70
|
+
See `Dockerfile.example` for an example.
|
71
|
+
|
72
|
+
#### Authentication
|
73
|
+
|
74
|
+
To authenticate to the Kubernetes API, you have a couple of options:
|
75
|
+
|
76
|
+
```bash
|
77
|
+
# Authentication option 1: HTTP basic auth
|
78
|
+
ENV KUBERNETES_API_USERNAME myusername
|
79
|
+
ENV KUBERNETES_API_PASSWORD mypassword
|
80
|
+
|
81
|
+
# Authentication option 2: client certificate
|
82
|
+
ENV KUBERNETES_API_CLIENT_KEY myclientkey
|
83
|
+
ENV KUBERNETES_API_CLIENT_CERT myclientcert
|
84
|
+
ENV KUBERNETES_API_CA_FILE path/to/my/ca/file
|
85
|
+
```
|
86
|
+
|
87
|
+
#### Dry run
|
88
|
+
|
89
|
+
You can put Reschedule in dry run mode with:
|
90
|
+
|
91
|
+
```bash
|
92
|
+
ENV RESCHEDULE_DRY_RUN 1
|
93
|
+
```
|
94
|
+
|
95
|
+
In this mode, Reschedule won't perform any rescheduling, but it will log the rescheduling that it would do.
|
96
|
+
|
7
97
|
License
|
8
98
|
-------
|
9
99
|
|
data/bin/reschedule
ADDED
data/docs/logo.png
ADDED
Binary file
|
data/lib/reschedule.rb
CHANGED
@@ -1,5 +1,28 @@
|
|
1
|
+
require 'active_support/all'
|
2
|
+
require 'httparty'
|
3
|
+
require 'kubeclient'
|
4
|
+
require 'rufus-scheduler'
|
5
|
+
|
1
6
|
directory = File.dirname(File.absolute_path(__FILE__))
|
2
|
-
Dir.glob("#{directory}/reschedule
|
7
|
+
Dir.glob("#{directory}/reschedule/reschedulers/base.rb") { |file| require file }
|
8
|
+
Dir.glob("#{directory}/reschedule/**/*.rb") { |file| require file }
|
3
9
|
|
4
10
|
module Reschedule
|
11
|
+
module_function
|
12
|
+
|
13
|
+
def self.configure
|
14
|
+
yield(configuration) if block_given?
|
15
|
+
end
|
16
|
+
|
17
|
+
def self.configuration
|
18
|
+
@configuration ||= Configuration.new
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.logger
|
22
|
+
@logger ||= begin
|
23
|
+
logger = Logger.new(STDOUT)
|
24
|
+
logger.progname = 'Reschedule'
|
25
|
+
logger
|
26
|
+
end
|
27
|
+
end
|
5
28
|
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Reschedule
|
2
|
+
class Configuration
|
3
|
+
attr_accessor :dry_run, :kubernetes_api_url, :kubernetes_api_username, :kubernetes_api_password,
|
4
|
+
:kubernetes_api_client_key, :kubernetes_api_client_cert, :kubernetes_api_ca_file
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
self.dry_run = ENV['RESCHEDULE_DRY_RUN'] == '1'
|
8
|
+
self.kubernetes_api_url = ENV['KUBERNETES_API_URL']
|
9
|
+
self.kubernetes_api_username = ENV['KUBERNETES_API_USERNAME']
|
10
|
+
self.kubernetes_api_password = ENV['KUBERNETES_API_PASSWORD']
|
11
|
+
self.kubernetes_api_client_key = ENV['KUBERNETES_API_CLIENT_KEY']
|
12
|
+
self.kubernetes_api_client_cert = ENV['KUBERNETES_API_CLIENT_CERT']
|
13
|
+
self.kubernetes_api_ca_file = ENV['KUBERNETES_API_CA_FILE']
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Reschedule
|
2
|
+
module Heapster
|
3
|
+
class Api
|
4
|
+
def get_node_stats(node_name)
|
5
|
+
get("model/nodes/#{node_name}/stats/")
|
6
|
+
end
|
7
|
+
|
8
|
+
def get_nodes
|
9
|
+
get('model/nodes/')
|
10
|
+
end
|
11
|
+
|
12
|
+
private
|
13
|
+
|
14
|
+
def get(path)
|
15
|
+
auth = {
|
16
|
+
username: Reschedule.configuration.kubernetes_api_username,
|
17
|
+
password: Reschedule.configuration.kubernetes_api_password
|
18
|
+
}
|
19
|
+
base_url = "#{Reschedule.configuration.kubernetes_api_url}v1/proxy/namespaces/kube-system/services/heapster/api/v1/"
|
20
|
+
url = "#{base_url}#{path}"
|
21
|
+
HTTParty.get(url, basic_auth: auth, verify: false)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
module Reschedule
|
2
|
+
module Kubernetes
|
3
|
+
class Api
|
4
|
+
attr_accessor :client
|
5
|
+
|
6
|
+
def initialize
|
7
|
+
self.client = Kubeclient::Client.new(Reschedule.configuration.kubernetes_api_url, 'v1', client_options)
|
8
|
+
end
|
9
|
+
|
10
|
+
def method_missing(method, *args)
|
11
|
+
self.client.public_send(method, *args)
|
12
|
+
end
|
13
|
+
|
14
|
+
private
|
15
|
+
|
16
|
+
def client_options
|
17
|
+
client_options = {}
|
18
|
+
if Reschedule.configuration.kubernetes_api_username.present?
|
19
|
+
client_options[:auth_options] = {
|
20
|
+
user: Reschedule.configuration.kubernetes_api_username,
|
21
|
+
password: Reschedule.configuration.kubernetes_api_password
|
22
|
+
}
|
23
|
+
end
|
24
|
+
if Reschedule.configuration.kubernetes_api_client_key.present?
|
25
|
+
client_options[:ssl_options] = {
|
26
|
+
client_key: Reschedule.configuration.kubernetes_api_client_key,
|
27
|
+
client_cert: Reschedule.configuration.kubernetes_api_client_cert,
|
28
|
+
ca_file: Reschedule.configuration.kubernetes_api_ca_file,
|
29
|
+
verify_ssl: OpenSSL::SSL::VERIFY_PEER
|
30
|
+
}
|
31
|
+
end
|
32
|
+
client_options
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
module Reschedule
|
2
|
+
class Manager
|
3
|
+
def run
|
4
|
+
scheduler = Rufus::Scheduler.new
|
5
|
+
|
6
|
+
config['reschedulers'].each do |rescheduler|
|
7
|
+
klass = "Reschedule::Reschedulers::#{rescheduler['type']}".constantize
|
8
|
+
scheduler.every rescheduler['every'] do
|
9
|
+
klass.new(rescheduler['options']).run
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
scheduler.join
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def config
|
19
|
+
@config ||= YAML.load(File.read('./reschedule.yml'))
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Reschedule
|
2
|
+
module Reschedulers
|
3
|
+
class All < Base
|
4
|
+
attr_reader :kubernetes_api, :replication_controller_name_regex
|
5
|
+
|
6
|
+
self.defaults = {
|
7
|
+
'namespace' => 'default'
|
8
|
+
}
|
9
|
+
|
10
|
+
def initialize(options={})
|
11
|
+
super
|
12
|
+
@kubernetes_api = Reschedule::Kubernetes::Api.new
|
13
|
+
@replication_controller_name_regex = options['replication_controller_name_match'].present? ? Regexp.new(options['replication_controller_name_match']) : nil
|
14
|
+
end
|
15
|
+
|
16
|
+
def run
|
17
|
+
super
|
18
|
+
replication_controllers = kubernetes_api.get_replication_controllers(namespace: options['namespace'])
|
19
|
+
Reschedule.logger.debug "Found #{replication_controllers.length} replication controllers"
|
20
|
+
|
21
|
+
replication_controllers.select! do |replication_controller|
|
22
|
+
template = replication_controller.spec[:template] || replication_controller.spec['table'][:template]
|
23
|
+
image = template['spec']['containers'][0]['image']
|
24
|
+
next if image.include?('/reschedule')
|
25
|
+
next if replication_controller.spec.replicas == 0
|
26
|
+
next if replication_controller_name_regex && replication_controller.metadata.name !~ replication_controller_name_regex
|
27
|
+
true
|
28
|
+
end
|
29
|
+
|
30
|
+
Reschedule.logger.debug "Found #{replication_controllers.length} replication controllers to reschedule"
|
31
|
+
replication_controllers.each do |replication_controller|
|
32
|
+
reschedule_replication_controller(replication_controller.metadata.name)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module Reschedule
|
2
|
+
module Reschedulers
|
3
|
+
class Base
|
4
|
+
attr_reader :options
|
5
|
+
|
6
|
+
class_attribute :defaults
|
7
|
+
|
8
|
+
def initialize(options={})
|
9
|
+
@options = options || {}
|
10
|
+
@options.reverse_merge!(self.class.defaults)
|
11
|
+
end
|
12
|
+
|
13
|
+
def run
|
14
|
+
Reschedule.logger.debug "Starting #{self.class.name}"
|
15
|
+
Reschedule.logger.debug "Dry run mode is on" if Reschedule.configuration.dry_run
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
def reschedule_replication_controller(replication_controller_name)
|
21
|
+
Reschedule.logger.debug "Rescheduling #{replication_controller_name}"
|
22
|
+
|
23
|
+
unless Reschedule.configuration.dry_run
|
24
|
+
replication_controller = kubernetes_api.get_replication_controllers(label_selector: "name=#{replication_controller_name}").first
|
25
|
+
original_replicas = replication_controller.spec.replicas
|
26
|
+
replication_controller.spec.replicas = 0
|
27
|
+
kubernetes_api.update_replication_controller(replication_controller)
|
28
|
+
sleep 0.5
|
29
|
+
replication_controller = kubernetes_api.get_replication_controllers(label_selector: "name=#{replication_controller_name}").first
|
30
|
+
replication_controller.spec.replicas = original_replicas
|
31
|
+
kubernetes_api.update_replication_controller(replication_controller)
|
32
|
+
end
|
33
|
+
|
34
|
+
Reschedule.logger.debug "Rescheduled #{replication_controller_name}"
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
module Reschedule
|
2
|
+
module Reschedulers
|
3
|
+
class MemoryThreshold < Base
|
4
|
+
attr_reader :heapster_api, :kubernetes_api
|
5
|
+
|
6
|
+
self.defaults = {
|
7
|
+
'memory_threshold' => 0.8,
|
8
|
+
'namespace' => 'default'
|
9
|
+
}
|
10
|
+
|
11
|
+
def initialize(options={})
|
12
|
+
super
|
13
|
+
@heapster_api = Reschedule::Heapster::Api.new
|
14
|
+
@kubernetes_api = Reschedule::Kubernetes::Api.new
|
15
|
+
end
|
16
|
+
|
17
|
+
def run
|
18
|
+
super
|
19
|
+
node_names = rescheduleable_node_names
|
20
|
+
Reschedule.logger.debug "Found #{node_names.length} rescheduleable nodes"
|
21
|
+
return if node_names.blank?
|
22
|
+
|
23
|
+
pods = get_pods
|
24
|
+
Reschedule.logger.debug "Found #{pods.length} total pods"
|
25
|
+
replication_controller_names = pods_and_node_names_to_replication_controller_names(pods, node_names)
|
26
|
+
replication_controller_names.each do |replication_controller_name|
|
27
|
+
reschedule_replication_controller(replication_controller_name)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def get_pods
|
34
|
+
pods = kubernetes_api.get_pods
|
35
|
+
pods.select { |pod| pod.metadata.namespace == options['namespace'] }
|
36
|
+
end
|
37
|
+
|
38
|
+
def pods_and_node_names_to_replication_controller_names(pods, node_names)
|
39
|
+
node_pods = pods.select do |pod|
|
40
|
+
node_names.include?(pod.spec.nodeName)
|
41
|
+
end
|
42
|
+
pods_to_replication_controller_names(node_pods)
|
43
|
+
end
|
44
|
+
|
45
|
+
def pods_to_replication_controller_names(pods)
|
46
|
+
replication_controller_names = pods.map do |pod|
|
47
|
+
created_by = pod.metadata.annotations['kubernetes.io/created-by']
|
48
|
+
created_by = JSON.parse(created_by)
|
49
|
+
created_by['reference']['name']
|
50
|
+
end
|
51
|
+
replication_controller_names.uniq
|
52
|
+
end
|
53
|
+
|
54
|
+
def rescheduleable_node_names
|
55
|
+
nodes = heapster_api.get_nodes
|
56
|
+
if nodes.code != 200
|
57
|
+
Reschedule.logger.debug "Unable to get nodes in Heapster: #{nodes}"
|
58
|
+
return
|
59
|
+
end
|
60
|
+
Reschedule.logger.debug "Found #{nodes.length} nodes in Heapster"
|
61
|
+
nodes_to_rescheduleable_node_names(nodes)
|
62
|
+
end
|
63
|
+
|
64
|
+
def nodes_to_rescheduleable_node_names(nodes)
|
65
|
+
rescheduleable_node_names = []
|
66
|
+
nodes.each do |node|
|
67
|
+
node_name = node['name']
|
68
|
+
node_stats = heapster_api.get_node_stats(node_name)
|
69
|
+
if node_stats.code != 200
|
70
|
+
Reschedule.logger.debug "Node stats not found: #{node_stats}"
|
71
|
+
next
|
72
|
+
end
|
73
|
+
memory_limit = node_stats['stats']['memory-limit']['hour']['average']
|
74
|
+
memory_usage = node_stats['stats']['memory-usage']['hour']['average']
|
75
|
+
if memory_usage.to_f / memory_limit > options['memory_threshold']
|
76
|
+
rescheduleable_node_names << node_name
|
77
|
+
end
|
78
|
+
end
|
79
|
+
rescheduleable_node_names
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
data/lib/reschedule/version.rb
CHANGED
data/reschedule.gemspec
CHANGED
@@ -3,8 +3,8 @@ require File.expand_path('../lib/reschedule/version', __FILE__)
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.authors = ['Tom Benner']
|
5
5
|
s.email = ['tombenner@gmail.com']
|
6
|
-
s.description = s.summary = %q{
|
7
|
-
s.homepage = 'https://github.com/
|
6
|
+
s.description = s.summary = %q{Automatic, configurable Kubernetes rescheduling}
|
7
|
+
s.homepage = 'https://github.com/entelo/reschedule'
|
8
8
|
|
9
9
|
s.files = `git ls-files`.split($\)
|
10
10
|
s.name = 'reschedule'
|
@@ -12,5 +12,12 @@ Gem::Specification.new do |s|
|
|
12
12
|
s.version = Reschedule::VERSION
|
13
13
|
s.license = 'MIT'
|
14
14
|
|
15
|
-
s.
|
15
|
+
s.add_dependency 'activesupport', '~> 4.2'
|
16
|
+
s.add_dependency 'httparty', '~> 0.13'
|
17
|
+
s.add_dependency 'kubeclient', '~> 0.9.0'
|
18
|
+
s.add_dependency 'rufus-scheduler', '~> 3.2'
|
19
|
+
|
20
|
+
s.add_development_dependency 'hashie', '~> 3.4'
|
21
|
+
s.add_development_dependency 'rspec', '~> 3.0'
|
22
|
+
s.add_development_dependency 'webmock', '~> 1.8'
|
16
23
|
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Reschedule::Reschedulers::All do
|
4
|
+
include RescheduleSpecHelper
|
5
|
+
|
6
|
+
describe '#run' do
|
7
|
+
let(:reschedule_replication_controller) { create_replication_controller('foo.bar.com/reschedule:latest') }
|
8
|
+
let(:other_replication_controller) { create_replication_controller('foo.bar.com/other:latest') }
|
9
|
+
|
10
|
+
before :each do
|
11
|
+
allow_any_instance_of(Reschedule::Kubernetes::Api).to receive(:get_replication_controllers).and_return(replication_controllers)
|
12
|
+
allow_any_instance_of(Reschedule::Kubernetes::Api).to receive(:update_replication_controller)
|
13
|
+
allow_any_instance_of(Reschedule::Kubernetes::Api).to receive(:get_pods).and_return(pods)
|
14
|
+
end
|
15
|
+
|
16
|
+
context 'default' do
|
17
|
+
context 'other replication controller' do
|
18
|
+
let(:replication_controllers) { [other_replication_controller] }
|
19
|
+
|
20
|
+
it 'updates all of the replication controllers' do
|
21
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to receive(:update_replication_controller).with(replication_controllers[0])
|
22
|
+
described_class.new.run
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
context 'reschedule replication controller' do
|
27
|
+
let(:replication_controllers) { [reschedule_replication_controller] }
|
28
|
+
|
29
|
+
it 'does not update any replication controller' do
|
30
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to_not receive(:update_replication_controller)
|
31
|
+
described_class.new.run
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
context 'dry run' do
|
37
|
+
let(:dry_run) { true }
|
38
|
+
|
39
|
+
it 'does not update any replication controllers' do
|
40
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to_not receive(:update_replication_controller)
|
41
|
+
described_class.new.run
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,53 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Reschedule::Reschedulers::MemoryThreshold do
|
4
|
+
include RescheduleSpecHelper
|
5
|
+
|
6
|
+
describe '#run' do
|
7
|
+
before :each do
|
8
|
+
stub_json(:get, "#{heapster_url}model/nodes/", nodes)
|
9
|
+
stub_json(:get, "#{heapster_url}model/nodes/#{node_names[0]}/stats/", node_stats[0])
|
10
|
+
stub_json(:get, "#{heapster_url}model/nodes/#{node_names[1]}/stats/", node_stats[1])
|
11
|
+
|
12
|
+
allow_any_instance_of(Reschedule::Kubernetes::Api).to receive(:get_replication_controllers).with(label_selector: "name=#{replication_controller_names[0]}").and_return(replication_controllers)
|
13
|
+
allow_any_instance_of(Reschedule::Kubernetes::Api).to receive(:update_replication_controller)
|
14
|
+
allow_any_instance_of(Reschedule::Kubernetes::Api).to receive(:get_pods).and_return(pods)
|
15
|
+
end
|
16
|
+
|
17
|
+
context 'memory threshold not exceeded' do
|
18
|
+
it 'does not update any replication controllers' do
|
19
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to_not receive(:update_replication_controller)
|
20
|
+
described_class.new.run
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
context 'memory threshold exceeded' do
|
25
|
+
it 'updates the replication controller' do
|
26
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to receive(:update_replication_controller).with(replication_controllers[0])
|
27
|
+
described_class.new('memory_threshold' => 0.01).run
|
28
|
+
end
|
29
|
+
|
30
|
+
context 'dry run' do
|
31
|
+
let(:dry_run) { true }
|
32
|
+
|
33
|
+
it 'does not update any replication controllers' do
|
34
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to_not receive(:update_replication_controller)
|
35
|
+
described_class.new.run
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
context 'memory threshold exceeded in a non-default namespace' do
|
41
|
+
let(:pods) do
|
42
|
+
pod = pod_default
|
43
|
+
pod.metadata.namespace = 'foo'
|
44
|
+
[pod]
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'does not update any replication controllers' do
|
48
|
+
expect_any_instance_of(Reschedule::Kubernetes::Api).to_not receive(:update_replication_controller)
|
49
|
+
described_class.new('memory_threshold' => 0.01).run
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
data/spec/spec_helper.rb
CHANGED
@@ -0,0 +1,106 @@
|
|
1
|
+
module RescheduleSpecHelper
|
2
|
+
extend RSpec::SharedContext
|
3
|
+
|
4
|
+
def stub_json(method, url, response_data)
|
5
|
+
response = {
|
6
|
+
status: 200,
|
7
|
+
body: response_data.to_json,
|
8
|
+
headers: { 'Content-Type' => 'application/json' }
|
9
|
+
}
|
10
|
+
stub_request(:get, url).to_return(response)
|
11
|
+
end
|
12
|
+
|
13
|
+
def create_replication_controller(image)
|
14
|
+
Hashie::Mash.new(
|
15
|
+
metadata: {
|
16
|
+
name: 'foo',
|
17
|
+
spec: {
|
18
|
+
replicas: 1
|
19
|
+
}
|
20
|
+
},
|
21
|
+
spec: {
|
22
|
+
table: {
|
23
|
+
template: {
|
24
|
+
spec: {
|
25
|
+
containers: [
|
26
|
+
{
|
27
|
+
image: image
|
28
|
+
}
|
29
|
+
]
|
30
|
+
}
|
31
|
+
}
|
32
|
+
}
|
33
|
+
}
|
34
|
+
)
|
35
|
+
end
|
36
|
+
|
37
|
+
before :each do
|
38
|
+
Reschedule.configure do |config|
|
39
|
+
config.kubernetes_api_url = "https://#{kube_host}/api/"
|
40
|
+
config.kubernetes_api_username = kube_username
|
41
|
+
config.kubernetes_api_password = kube_password
|
42
|
+
config.dry_run = dry_run
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
let(:kube_host) { 'kube.local' }
|
47
|
+
let(:kube_username) { 'foo' }
|
48
|
+
let(:kube_password) { 'bar' }
|
49
|
+
let(:kube_url) { "https://#{kube_username}:#{kube_password}@#{kube_host}/api/" }
|
50
|
+
let(:heapster_url) { "#{kube_url}v1/proxy/namespaces/kube-system/services/heapster/api/v1/" }
|
51
|
+
let(:dry_run) { false }
|
52
|
+
|
53
|
+
let(:node_names) { (0..1).map { |i| "node-#{i}.compute.internal" } }
|
54
|
+
let(:replication_controller_names) { (0..1).map { |i| "rc-#{i}" } }
|
55
|
+
|
56
|
+
let(:node_stats) { [node_stat_default, node_stat_default] }
|
57
|
+
let(:nodes) do
|
58
|
+
(0..1).map do |i|
|
59
|
+
{
|
60
|
+
'name' => node_names[i],
|
61
|
+
'cpuUsage' => 306,
|
62
|
+
'memUsage' => 2831523840
|
63
|
+
}
|
64
|
+
end
|
65
|
+
end
|
66
|
+
let(:pods) { [pod_default] }
|
67
|
+
let(:replication_controllers) { [double.as_null_object] }
|
68
|
+
|
69
|
+
let(:node_stat_default) do
|
70
|
+
{
|
71
|
+
"uptime"=>401576,
|
72
|
+
"stats"=>
|
73
|
+
{"cpu-limit"=>
|
74
|
+
{"minute"=>{"average"=>36000, "percentile"=>36000, "max"=>36000},
|
75
|
+
"hour"=>{"average"=>36000, "percentile"=>36000, "max"=>36000},
|
76
|
+
"day"=>{"average"=>36000, "percentile"=>36000, "max"=>36000}},
|
77
|
+
"cpu-usage"=>
|
78
|
+
{"minute"=>{"average"=>306, "percentile"=>306, "max"=>306}, "hour"=>{"average"=>317, "percentile"=>340, "max"=>350}, "day"=>{"average"=>316, "percentile"=>330, "max"=>330}},
|
79
|
+
"memory-limit"=>
|
80
|
+
{"minute"=>{"average"=>63317110784, "percentile"=>63317110784, "max"=>63317110784},
|
81
|
+
"hour"=>{"average"=>63317213184, "percentile"=>63317213184, "max"=>63317213184},
|
82
|
+
"day"=>{"average"=>63317213184, "percentile"=>63317213184, "max"=>63317213184}},
|
83
|
+
"memory-usage"=>
|
84
|
+
{"minute"=>{"average"=>4992581632, "percentile"=>4992581632, "max"=>4992581632},
|
85
|
+
"hour"=>{"average"=>4982343816, "percentile"=>4991221760, "max"=>4995416064},
|
86
|
+
"day"=>{"average"=>4968991948, "percentile"=>4978638848, "max"=>4979036160}},
|
87
|
+
"memory-working"=>
|
88
|
+
{"minute"=>{"average"=>2815111168, "percentile"=>2815111168, "max"=>2815111168},
|
89
|
+
"hour"=>{"average"=>2810043869, "percentile"=>2814377984, "max"=>2818572288},
|
90
|
+
"day"=>{"average"=>2805639850, "percentile"=>2810183680, "max"=>2810183680}}}}
|
91
|
+
end
|
92
|
+
|
93
|
+
let(:pod_default) do
|
94
|
+
Hashie::Mash.new({
|
95
|
+
spec: { nodeName: node_names[0] },
|
96
|
+
metadata: {
|
97
|
+
annotations: {
|
98
|
+
'kubernetes.io/created-by' => {
|
99
|
+
reference: { name: replication_controller_names[0] }
|
100
|
+
}.to_json
|
101
|
+
},
|
102
|
+
namespace: 'default'
|
103
|
+
}
|
104
|
+
})
|
105
|
+
end
|
106
|
+
end
|
metadata
CHANGED
@@ -1,30 +1,114 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: reschedule
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tom Benner
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-05-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: activesupport
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '4.2'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '4.2'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: httparty
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0.13'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0.13'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: kubeclient
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: 0.9.0
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: 0.9.0
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: rufus-scheduler
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '3.2'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '3.2'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: hashie
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - "~>"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '3.4'
|
76
|
+
type: :development
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - "~>"
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '3.4'
|
13
83
|
- !ruby/object:Gem::Dependency
|
14
84
|
name: rspec
|
15
85
|
requirement: !ruby/object:Gem::Requirement
|
16
86
|
requirements:
|
17
|
-
- - "
|
87
|
+
- - "~>"
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '3.0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - "~>"
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '3.0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: webmock
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - "~>"
|
18
102
|
- !ruby/object:Gem::Version
|
19
|
-
version: '
|
103
|
+
version: '1.8'
|
20
104
|
type: :development
|
21
105
|
prerelease: false
|
22
106
|
version_requirements: !ruby/object:Gem::Requirement
|
23
107
|
requirements:
|
24
|
-
- - "
|
108
|
+
- - "~>"
|
25
109
|
- !ruby/object:Gem::Version
|
26
|
-
version: '
|
27
|
-
description:
|
110
|
+
version: '1.8'
|
111
|
+
description: Automatic, configurable Kubernetes rescheduling
|
28
112
|
email:
|
29
113
|
- tombenner@gmail.com
|
30
114
|
executables: []
|
@@ -32,15 +116,29 @@ extensions: []
|
|
32
116
|
extra_rdoc_files: []
|
33
117
|
files:
|
34
118
|
- ".gitignore"
|
119
|
+
- Dockerfile.example
|
35
120
|
- Gemfile
|
36
121
|
- MIT-LICENSE
|
37
122
|
- README.md
|
38
123
|
- Rakefile
|
124
|
+
- bin/reschedule
|
125
|
+
- docs/logo.png
|
39
126
|
- lib/reschedule.rb
|
127
|
+
- lib/reschedule/configuration.rb
|
128
|
+
- lib/reschedule/heapster/api.rb
|
129
|
+
- lib/reschedule/kubernetes/api.rb
|
130
|
+
- lib/reschedule/manager.rb
|
131
|
+
- lib/reschedule/reschedulers/all.rb
|
132
|
+
- lib/reschedule/reschedulers/base.rb
|
133
|
+
- lib/reschedule/reschedulers/memory_threshold.rb
|
40
134
|
- lib/reschedule/version.rb
|
41
135
|
- reschedule.gemspec
|
136
|
+
- reschedule.yml.example
|
137
|
+
- spec/lib/reschedule/reschedulers/all_spec.rb
|
138
|
+
- spec/lib/reschedule/reschedulers/memory_threshold_spec.rb
|
42
139
|
- spec/spec_helper.rb
|
43
|
-
|
140
|
+
- spec/support/reschedule_spec_helper.rb
|
141
|
+
homepage: https://github.com/entelo/reschedule
|
44
142
|
licenses:
|
45
143
|
- MIT
|
46
144
|
metadata: {}
|
@@ -63,5 +161,5 @@ rubyforge_project:
|
|
63
161
|
rubygems_version: 2.4.8
|
64
162
|
signing_key:
|
65
163
|
specification_version: 4
|
66
|
-
summary:
|
164
|
+
summary: Automatic, configurable Kubernetes rescheduling
|
67
165
|
test_files: []
|