resque-kubernetes 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 7f5cf6f4a4069b9440bbd5a136a24080214670c2
4
- data.tar.gz: fac4fd9e289b57cce831e006459828481128c542
3
+ metadata.gz: e0881cb0e587e677806dbf25580ac7c22de17d29
4
+ data.tar.gz: 57a3680a1a064f9ac0d16b11c29c0bc5948862af
5
5
  SHA512:
6
- metadata.gz: c36a6aafdb792d2b1cc1e3e137d6cc18a2f9cb0f3f0e8f1fc2ae9ee56489ea5688b1a9c4148fb92ea9d6accf0ccc9fd24ede2fd3ae132220a5ecbb7580ebdeb2
7
- data.tar.gz: 365a4a5f2abeab6b8ad6ff2a4c032586fc19ce188f1ce04b2540df678d9a9d216e0909abe5e08c234cf6001ebc550599fe3386fa55f57269a83a130d32197cf5
6
+ metadata.gz: a73a284922f6e304bea0a3da941586e96dec3edb0d035e991f7a16e52f3ca80c5c8dc11ebc9aebba3879f213f0680e3d5ca34885cb3f6b1677b22a7859a6e41d
7
+ data.tar.gz: 5aded9eb8bfaabee8568e0f8ca8f1e5632c7ebfe65c8cdca2ade39a22bdf0cc3468a733a65938c5b79aceb6763e01decbe772ae3d58c50ca667864b6cd5d125c
data/.rubocop.yml CHANGED
@@ -45,4 +45,4 @@ Lint/UselessSetterCall:
45
45
  Exclude:
46
46
  # Rubocop is incorrectly flagging `term_on_empty` as local
47
47
  # See: https://github.com/bbatsov/rubocop/issues/5420
48
- - lib/resque/kubernetes/job.rb
48
+ - lib/resque/kubernetes/jobs_manager.rb
data/CHANGELOG.md CHANGED
@@ -1,26 +1,32 @@
1
- # 0.8.0
1
+ # v0.9.0
2
+ - Update to not pollute the job class with our methods
3
+
4
+ # v0.8.0
2
5
  - Fix bug where enqueueing jobs would keep adding workers if worker count
3
6
  was _greater_ than `max_workers`
4
- # 0.7.0
7
+
8
+ # v0.7.0
5
9
  - Update to support kubeclient 2.2 or 3.x
6
- # 0.6.0
10
+
11
+ # v0.6.0
7
12
  - Add support for ActiveJob when configured to be backed by Resque
8
13
  - When authorizing with `~/.kube/config` use Google Default Application Credentials rather than require a
9
14
  forked version of `kubeclient`
10
- # 0.5.0
15
+
16
+ # v0.5.0
11
17
  - Maximum workers can no be configured per job type
12
18
  - Fix a crash when cleaning up a job that was removed by another process
13
19
  - No longer clean up pods because cleaning up finished jobs takes care of that
14
20
  - Apply rubocop, and bundler-audit rules
15
21
 
16
- # 0.4.0
22
+ # v0.4.0
17
23
  - Syntax error fix
18
24
 
19
- # 0.3.0
25
+ # v0.3.0
20
26
  - Syntax error fix
21
27
 
22
- # 0.2.0
28
+ # v0.2.0
23
29
  - Fix for running in GKE cluster and production Rails environment
24
30
 
25
- # 0.1.0
31
+ # v0.1.0
26
32
  - Initial release
@@ -1,7 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require "kubeclient"
4
-
5
3
  module Resque
6
4
  module Kubernetes
7
5
  # Resque hook to autoscale Kubernetes Jobs for workers.
@@ -89,13 +87,12 @@ module Resque
89
87
  return unless Resque::Kubernetes.environments.include?(Rails.env)
90
88
  end
91
89
 
92
- reap_finished_jobs
93
- apply_kubernetes_job
90
+ manager = JobsManager.new(self)
91
+ manager.reap_finished_jobs
92
+ manager.apply_kubernetes_job
94
93
  end
95
94
 
96
- protected
97
-
98
- # Return the maximum number of workers to autoscale the job to.
95
+ # The maximum number of workers to autoscale the job to.
99
96
  #
100
97
  # While the number of active Kubernetes Jobs is less than this number,
101
98
  # the gem will add new Jobs to auto-scale the workers.
@@ -118,105 +115,6 @@ module Resque
118
115
  def max_workers
119
116
  Resque::Kubernetes.max_workers
120
117
  end
121
-
122
- private
123
-
124
- def jobs_client
125
- return @jobs_client if @jobs_client
126
- @jobs_client = client("/apis/batch")
127
- end
128
-
129
- def client(scope)
130
- context = ContextFactory.context
131
- return unless context
132
-
133
- Kubeclient::Client.new(
134
- context.api_endpoint + scope,
135
- context.api_version,
136
- ssl_options: context.ssl_options,
137
- auth_options: context.auth_options
138
- )
139
- end
140
-
141
- def finished_jobs
142
- resque_jobs = jobs_client.get_jobs(label_selector: "resque-kubernetes=job")
143
- resque_jobs.select { |job| job.spec.completions == job.status.succeeded }
144
- end
145
-
146
- def reap_finished_jobs
147
- finished_jobs.each do |job|
148
- begin
149
- jobs_client.delete_job(job.metadata.name, job.metadata.namespace)
150
- rescue KubeException => e
151
- raise unless e.error_code == 404
152
- end
153
- end
154
- end
155
-
156
- def apply_kubernetes_job
157
- manifest = DeepHash.new.merge!(job_manifest)
158
- ensure_namespace(manifest)
159
-
160
- # Do not start job if we have reached our maximum count
161
- return if jobs_maxed?(manifest["metadata"]["name"], manifest["metadata"]["namespace"])
162
-
163
- adjust_manifest(manifest)
164
-
165
- job = Kubeclient::Resource.new(manifest)
166
- jobs_client.create_job(job)
167
- end
168
-
169
- def jobs_maxed?(name, namespace)
170
- resque_jobs = jobs_client.get_jobs(
171
- label_selector: "resque-kubernetes=job,resque-kubernetes-group=#{name}",
172
- namespace: namespace
173
- )
174
- running = resque_jobs.reject { |job| job.spec.completions == job.status.succeeded }
175
- running.size >= max_workers
176
- end
177
-
178
- def adjust_manifest(manifest)
179
- add_labels(manifest)
180
- ensure_term_on_empty(manifest)
181
- ensure_reset_policy(manifest)
182
- update_job_name(manifest)
183
- end
184
-
185
- def add_labels(manifest)
186
- manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
187
- manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
188
- manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
189
- end
190
-
191
- def ensure_term_on_empty(manifest)
192
- manifest["spec"]["template"]["spec"] ||= {}
193
- manifest["spec"]["template"]["spec"]["containers"] ||= []
194
- manifest["spec"]["template"]["spec"]["containers"].each do |container|
195
- container_term_on_empty(container)
196
- end
197
- end
198
-
199
- def container_term_on_empty(container)
200
- container["env"] ||= []
201
- term_on_empty = container["env"].find { |env| env["name"] == "TERM_ON_EMPTY" }
202
- unless term_on_empty
203
- term_on_empty = {"name" => "TERM_ON_EMPTY"}
204
- container["env"] << term_on_empty
205
- end
206
- term_on_empty["value"] = "1"
207
- end
208
-
209
- def ensure_reset_policy(manifest)
210
- manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
211
- end
212
-
213
- def ensure_namespace(manifest)
214
- manifest["metadata"]["namespace"] ||= "default"
215
- end
216
-
217
- def update_job_name(manifest)
218
- manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
219
- end
220
118
  end
221
119
  end
222
120
  end
@@ -0,0 +1,116 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "kubeclient"
4
+
5
+ module Resque
6
+ module Kubernetes
7
+ # Spins up Kubernetes Jobs to run Resque workers.
8
+ class JobsManager
9
+ attr_reader :owner
10
+ private :owner
11
+
12
+ def initialize(owner)
13
+ @owner = owner
14
+ end
15
+
16
+ def reap_finished_jobs
17
+ finished_jobs.each do |job|
18
+ begin
19
+ jobs_client.delete_job(job.metadata.name, job.metadata.namespace)
20
+ rescue KubeException => e
21
+ raise unless e.error_code == 404
22
+ end
23
+ end
24
+ end
25
+
26
+ def apply_kubernetes_job
27
+ manifest = DeepHash.new.merge!(owner.job_manifest)
28
+ ensure_namespace(manifest)
29
+
30
+ # Do not start job if we have reached our maximum count
31
+ return if jobs_maxed?(manifest["metadata"]["name"], manifest["metadata"]["namespace"])
32
+
33
+ adjust_manifest(manifest)
34
+
35
+ job = Kubeclient::Resource.new(manifest)
36
+ jobs_client.create_job(job)
37
+ end
38
+
39
+ private
40
+
41
+ def jobs_client
42
+ return @jobs_client if @jobs_client
43
+ @jobs_client = client("/apis/batch")
44
+ end
45
+
46
+ def client(scope)
47
+ context = ContextFactory.context
48
+ return unless context
49
+
50
+ Kubeclient::Client.new(
51
+ context.api_endpoint + scope,
52
+ context.api_version,
53
+ ssl_options: context.ssl_options,
54
+ auth_options: context.auth_options
55
+ )
56
+ end
57
+
58
+ def finished_jobs
59
+ resque_jobs = jobs_client.get_jobs(label_selector: "resque-kubernetes=job")
60
+ resque_jobs.select { |job| job.spec.completions == job.status.succeeded }
61
+ end
62
+
63
+ def jobs_maxed?(name, namespace)
64
+ resque_jobs = jobs_client.get_jobs(
65
+ label_selector: "resque-kubernetes=job,resque-kubernetes-group=#{name}",
66
+ namespace: namespace
67
+ )
68
+ running = resque_jobs.reject { |job| job.spec.completions == job.status.succeeded }
69
+ running.size >= owner.max_workers
70
+ end
71
+
72
+ def adjust_manifest(manifest)
73
+ add_labels(manifest)
74
+ ensure_term_on_empty(manifest)
75
+ ensure_reset_policy(manifest)
76
+ update_job_name(manifest)
77
+ end
78
+
79
+ def add_labels(manifest)
80
+ manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
81
+ manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
82
+ manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
83
+ end
84
+
85
+ def ensure_term_on_empty(manifest)
86
+ manifest["spec"]["template"]["spec"] ||= {}
87
+ manifest["spec"]["template"]["spec"]["containers"] ||= []
88
+ manifest["spec"]["template"]["spec"]["containers"].each do |container|
89
+ container_term_on_empty(container)
90
+ end
91
+ end
92
+
93
+ def container_term_on_empty(container)
94
+ container["env"] ||= []
95
+ term_on_empty = container["env"].find { |env| env["name"] == "TERM_ON_EMPTY" }
96
+ unless term_on_empty
97
+ term_on_empty = {"name" => "TERM_ON_EMPTY"}
98
+ container["env"] << term_on_empty
99
+ end
100
+ term_on_empty["value"] = "1"
101
+ end
102
+
103
+ def ensure_reset_policy(manifest)
104
+ manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
105
+ end
106
+
107
+ def ensure_namespace(manifest)
108
+ manifest["metadata"]["namespace"] ||= "default"
109
+ end
110
+
111
+ def update_job_name(manifest)
112
+ manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
113
+ end
114
+ end
115
+ end
116
+ end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Resque
4
4
  module Kubernetes
5
- VERSION = "0.8.0"
5
+ VERSION = "0.9.0"
6
6
  end
7
7
  end
@@ -1,12 +1,13 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ require "resque/kubernetes/configurable"
3
4
  require "resque/kubernetes/context_factory"
4
5
  require "resque/kubernetes/deep_hash"
5
6
  require "resque/kubernetes/dns_safe_random"
6
7
  require "resque/kubernetes/job"
8
+ require "resque/kubernetes/jobs_manager"
7
9
  require "resque/kubernetes/version"
8
10
  require "resque/kubernetes/worker"
9
- require "resque/kubernetes/configurable"
10
11
 
11
12
  module Resque
12
13
  # Run Resque Jobs as Kubernetes Jobs with autoscaling.
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: resque-kubernetes
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.0
4
+ version: 0.9.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Wadsack
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-05-22 00:00:00.000000000 Z
11
+ date: 2018-06-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -160,6 +160,7 @@ files:
160
160
  - lib/resque/kubernetes/deep_hash.rb
161
161
  - lib/resque/kubernetes/dns_safe_random.rb
162
162
  - lib/resque/kubernetes/job.rb
163
+ - lib/resque/kubernetes/jobs_manager.rb
163
164
  - lib/resque/kubernetes/version.rb
164
165
  - lib/resque/kubernetes/worker.rb
165
166
  - resque-kubernetes.gemspec