resque-kubernetes 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +1 -1
- data/CHANGELOG.md +14 -8
- data/lib/resque/kubernetes/job.rb +4 -106
- data/lib/resque/kubernetes/jobs_manager.rb +116 -0
- data/lib/resque/kubernetes/version.rb +1 -1
- data/lib/resque/kubernetes.rb +2 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e0881cb0e587e677806dbf25580ac7c22de17d29
|
4
|
+
data.tar.gz: 57a3680a1a064f9ac0d16b11c29c0bc5948862af
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a73a284922f6e304bea0a3da941586e96dec3edb0d035e991f7a16e52f3ca80c5c8dc11ebc9aebba3879f213f0680e3d5ca34885cb3f6b1677b22a7859a6e41d
|
7
|
+
data.tar.gz: 5aded9eb8bfaabee8568e0f8ca8f1e5632c7ebfe65c8cdca2ade39a22bdf0cc3468a733a65938c5b79aceb6763e01decbe772ae3d58c50ca667864b6cd5d125c
|
data/.rubocop.yml
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,26 +1,32 @@
|
|
1
|
-
#
|
1
|
+
# v0.9.0
|
2
|
+
- Update to not pollute the job class with our methods
|
3
|
+
|
4
|
+
# v0.8.0
|
2
5
|
- Fix bug where enqueueing jobs would keep adding workers if worker count
|
3
6
|
was _greater_ than `max_workers`
|
4
|
-
|
7
|
+
|
8
|
+
# v0.7.0
|
5
9
|
- Update to support kubeclient 2.2 or 3.x
|
6
|
-
|
10
|
+
|
11
|
+
# v0.6.0
|
7
12
|
- Add support for ActiveJob when configured to be backed by Resque
|
8
13
|
- When authorizing with `~/.kube/config` use Google Default Application Credentials rather than require a
|
9
14
|
forked version of `kubeclient`
|
10
|
-
|
15
|
+
|
16
|
+
# v0.5.0
|
11
17
|
- Maximum workers can no be configured per job type
|
12
18
|
- Fix a crash when cleaning up a job that was removed by another process
|
13
19
|
- No longer clean up pods because cleaning up finished jobs takes care of that
|
14
20
|
- Apply rubocop, and bundler-audit rules
|
15
21
|
|
16
|
-
#
|
22
|
+
# v0.4.0
|
17
23
|
- Syntax error fix
|
18
24
|
|
19
|
-
#
|
25
|
+
# v0.3.0
|
20
26
|
- Syntax error fix
|
21
27
|
|
22
|
-
#
|
28
|
+
# v0.2.0
|
23
29
|
- Fix for running in GKE cluster and production Rails environment
|
24
30
|
|
25
|
-
#
|
31
|
+
# v0.1.0
|
26
32
|
- Initial release
|
@@ -1,7 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "kubeclient"
|
4
|
-
|
5
3
|
module Resque
|
6
4
|
module Kubernetes
|
7
5
|
# Resque hook to autoscale Kubernetes Jobs for workers.
|
@@ -89,13 +87,12 @@ module Resque
|
|
89
87
|
return unless Resque::Kubernetes.environments.include?(Rails.env)
|
90
88
|
end
|
91
89
|
|
92
|
-
|
93
|
-
|
90
|
+
manager = JobsManager.new(self)
|
91
|
+
manager.reap_finished_jobs
|
92
|
+
manager.apply_kubernetes_job
|
94
93
|
end
|
95
94
|
|
96
|
-
|
97
|
-
|
98
|
-
# Return the maximum number of workers to autoscale the job to.
|
95
|
+
# The maximum number of workers to autoscale the job to.
|
99
96
|
#
|
100
97
|
# While the number of active Kubernetes Jobs is less than this number,
|
101
98
|
# the gem will add new Jobs to auto-scale the workers.
|
@@ -118,105 +115,6 @@ module Resque
|
|
118
115
|
def max_workers
|
119
116
|
Resque::Kubernetes.max_workers
|
120
117
|
end
|
121
|
-
|
122
|
-
private
|
123
|
-
|
124
|
-
def jobs_client
|
125
|
-
return @jobs_client if @jobs_client
|
126
|
-
@jobs_client = client("/apis/batch")
|
127
|
-
end
|
128
|
-
|
129
|
-
def client(scope)
|
130
|
-
context = ContextFactory.context
|
131
|
-
return unless context
|
132
|
-
|
133
|
-
Kubeclient::Client.new(
|
134
|
-
context.api_endpoint + scope,
|
135
|
-
context.api_version,
|
136
|
-
ssl_options: context.ssl_options,
|
137
|
-
auth_options: context.auth_options
|
138
|
-
)
|
139
|
-
end
|
140
|
-
|
141
|
-
def finished_jobs
|
142
|
-
resque_jobs = jobs_client.get_jobs(label_selector: "resque-kubernetes=job")
|
143
|
-
resque_jobs.select { |job| job.spec.completions == job.status.succeeded }
|
144
|
-
end
|
145
|
-
|
146
|
-
def reap_finished_jobs
|
147
|
-
finished_jobs.each do |job|
|
148
|
-
begin
|
149
|
-
jobs_client.delete_job(job.metadata.name, job.metadata.namespace)
|
150
|
-
rescue KubeException => e
|
151
|
-
raise unless e.error_code == 404
|
152
|
-
end
|
153
|
-
end
|
154
|
-
end
|
155
|
-
|
156
|
-
def apply_kubernetes_job
|
157
|
-
manifest = DeepHash.new.merge!(job_manifest)
|
158
|
-
ensure_namespace(manifest)
|
159
|
-
|
160
|
-
# Do not start job if we have reached our maximum count
|
161
|
-
return if jobs_maxed?(manifest["metadata"]["name"], manifest["metadata"]["namespace"])
|
162
|
-
|
163
|
-
adjust_manifest(manifest)
|
164
|
-
|
165
|
-
job = Kubeclient::Resource.new(manifest)
|
166
|
-
jobs_client.create_job(job)
|
167
|
-
end
|
168
|
-
|
169
|
-
def jobs_maxed?(name, namespace)
|
170
|
-
resque_jobs = jobs_client.get_jobs(
|
171
|
-
label_selector: "resque-kubernetes=job,resque-kubernetes-group=#{name}",
|
172
|
-
namespace: namespace
|
173
|
-
)
|
174
|
-
running = resque_jobs.reject { |job| job.spec.completions == job.status.succeeded }
|
175
|
-
running.size >= max_workers
|
176
|
-
end
|
177
|
-
|
178
|
-
def adjust_manifest(manifest)
|
179
|
-
add_labels(manifest)
|
180
|
-
ensure_term_on_empty(manifest)
|
181
|
-
ensure_reset_policy(manifest)
|
182
|
-
update_job_name(manifest)
|
183
|
-
end
|
184
|
-
|
185
|
-
def add_labels(manifest)
|
186
|
-
manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
|
187
|
-
manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
|
188
|
-
manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
|
189
|
-
end
|
190
|
-
|
191
|
-
def ensure_term_on_empty(manifest)
|
192
|
-
manifest["spec"]["template"]["spec"] ||= {}
|
193
|
-
manifest["spec"]["template"]["spec"]["containers"] ||= []
|
194
|
-
manifest["spec"]["template"]["spec"]["containers"].each do |container|
|
195
|
-
container_term_on_empty(container)
|
196
|
-
end
|
197
|
-
end
|
198
|
-
|
199
|
-
def container_term_on_empty(container)
|
200
|
-
container["env"] ||= []
|
201
|
-
term_on_empty = container["env"].find { |env| env["name"] == "TERM_ON_EMPTY" }
|
202
|
-
unless term_on_empty
|
203
|
-
term_on_empty = {"name" => "TERM_ON_EMPTY"}
|
204
|
-
container["env"] << term_on_empty
|
205
|
-
end
|
206
|
-
term_on_empty["value"] = "1"
|
207
|
-
end
|
208
|
-
|
209
|
-
def ensure_reset_policy(manifest)
|
210
|
-
manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
|
211
|
-
end
|
212
|
-
|
213
|
-
def ensure_namespace(manifest)
|
214
|
-
manifest["metadata"]["namespace"] ||= "default"
|
215
|
-
end
|
216
|
-
|
217
|
-
def update_job_name(manifest)
|
218
|
-
manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
|
219
|
-
end
|
220
118
|
end
|
221
119
|
end
|
222
120
|
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "kubeclient"
|
4
|
+
|
5
|
+
module Resque
|
6
|
+
module Kubernetes
|
7
|
+
# Spins up Kubernetes Jobs to run Resque workers.
|
8
|
+
class JobsManager
|
9
|
+
attr_reader :owner
|
10
|
+
private :owner
|
11
|
+
|
12
|
+
def initialize(owner)
|
13
|
+
@owner = owner
|
14
|
+
end
|
15
|
+
|
16
|
+
def reap_finished_jobs
|
17
|
+
finished_jobs.each do |job|
|
18
|
+
begin
|
19
|
+
jobs_client.delete_job(job.metadata.name, job.metadata.namespace)
|
20
|
+
rescue KubeException => e
|
21
|
+
raise unless e.error_code == 404
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def apply_kubernetes_job
|
27
|
+
manifest = DeepHash.new.merge!(owner.job_manifest)
|
28
|
+
ensure_namespace(manifest)
|
29
|
+
|
30
|
+
# Do not start job if we have reached our maximum count
|
31
|
+
return if jobs_maxed?(manifest["metadata"]["name"], manifest["metadata"]["namespace"])
|
32
|
+
|
33
|
+
adjust_manifest(manifest)
|
34
|
+
|
35
|
+
job = Kubeclient::Resource.new(manifest)
|
36
|
+
jobs_client.create_job(job)
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def jobs_client
|
42
|
+
return @jobs_client if @jobs_client
|
43
|
+
@jobs_client = client("/apis/batch")
|
44
|
+
end
|
45
|
+
|
46
|
+
def client(scope)
|
47
|
+
context = ContextFactory.context
|
48
|
+
return unless context
|
49
|
+
|
50
|
+
Kubeclient::Client.new(
|
51
|
+
context.api_endpoint + scope,
|
52
|
+
context.api_version,
|
53
|
+
ssl_options: context.ssl_options,
|
54
|
+
auth_options: context.auth_options
|
55
|
+
)
|
56
|
+
end
|
57
|
+
|
58
|
+
def finished_jobs
|
59
|
+
resque_jobs = jobs_client.get_jobs(label_selector: "resque-kubernetes=job")
|
60
|
+
resque_jobs.select { |job| job.spec.completions == job.status.succeeded }
|
61
|
+
end
|
62
|
+
|
63
|
+
def jobs_maxed?(name, namespace)
|
64
|
+
resque_jobs = jobs_client.get_jobs(
|
65
|
+
label_selector: "resque-kubernetes=job,resque-kubernetes-group=#{name}",
|
66
|
+
namespace: namespace
|
67
|
+
)
|
68
|
+
running = resque_jobs.reject { |job| job.spec.completions == job.status.succeeded }
|
69
|
+
running.size >= owner.max_workers
|
70
|
+
end
|
71
|
+
|
72
|
+
def adjust_manifest(manifest)
|
73
|
+
add_labels(manifest)
|
74
|
+
ensure_term_on_empty(manifest)
|
75
|
+
ensure_reset_policy(manifest)
|
76
|
+
update_job_name(manifest)
|
77
|
+
end
|
78
|
+
|
79
|
+
def add_labels(manifest)
|
80
|
+
manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
|
81
|
+
manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
|
82
|
+
manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
|
83
|
+
end
|
84
|
+
|
85
|
+
def ensure_term_on_empty(manifest)
|
86
|
+
manifest["spec"]["template"]["spec"] ||= {}
|
87
|
+
manifest["spec"]["template"]["spec"]["containers"] ||= []
|
88
|
+
manifest["spec"]["template"]["spec"]["containers"].each do |container|
|
89
|
+
container_term_on_empty(container)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
def container_term_on_empty(container)
|
94
|
+
container["env"] ||= []
|
95
|
+
term_on_empty = container["env"].find { |env| env["name"] == "TERM_ON_EMPTY" }
|
96
|
+
unless term_on_empty
|
97
|
+
term_on_empty = {"name" => "TERM_ON_EMPTY"}
|
98
|
+
container["env"] << term_on_empty
|
99
|
+
end
|
100
|
+
term_on_empty["value"] = "1"
|
101
|
+
end
|
102
|
+
|
103
|
+
def ensure_reset_policy(manifest)
|
104
|
+
manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
|
105
|
+
end
|
106
|
+
|
107
|
+
def ensure_namespace(manifest)
|
108
|
+
manifest["metadata"]["namespace"] ||= "default"
|
109
|
+
end
|
110
|
+
|
111
|
+
def update_job_name(manifest)
|
112
|
+
manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
data/lib/resque/kubernetes.rb
CHANGED
@@ -1,12 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require "resque/kubernetes/configurable"
|
3
4
|
require "resque/kubernetes/context_factory"
|
4
5
|
require "resque/kubernetes/deep_hash"
|
5
6
|
require "resque/kubernetes/dns_safe_random"
|
6
7
|
require "resque/kubernetes/job"
|
8
|
+
require "resque/kubernetes/jobs_manager"
|
7
9
|
require "resque/kubernetes/version"
|
8
10
|
require "resque/kubernetes/worker"
|
9
|
-
require "resque/kubernetes/configurable"
|
10
11
|
|
11
12
|
module Resque
|
12
13
|
# Run Resque Jobs as Kubernetes Jobs with autoscaling.
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: resque-kubernetes
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeremy Wadsack
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-06-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -160,6 +160,7 @@ files:
|
|
160
160
|
- lib/resque/kubernetes/deep_hash.rb
|
161
161
|
- lib/resque/kubernetes/dns_safe_random.rb
|
162
162
|
- lib/resque/kubernetes/job.rb
|
163
|
+
- lib/resque/kubernetes/jobs_manager.rb
|
163
164
|
- lib/resque/kubernetes/version.rb
|
164
165
|
- lib/resque/kubernetes/worker.rb
|
165
166
|
- resque-kubernetes.gemspec
|