resque-kubernetes 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +4 -1
- data/CHANGELOG.md +3 -0
- data/README.md +1 -0
- data/lib/resque/kubernetes.rb +1 -0
- data/lib/resque/kubernetes/dns_safe_random.rb +4 -4
- data/lib/resque/kubernetes/job.rb +2 -1
- data/lib/resque/kubernetes/jobs_manager.rb +27 -43
- data/lib/resque/kubernetes/manifest_conformance.rb +52 -0
- data/lib/resque/kubernetes/version.rb +1 -1
- data/resque-kubernetes.gemspec +3 -2
- metadata +9 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bbfbc8c610d079165438b6a635043070730ecc0a
|
4
|
+
data.tar.gz: a409a4638079687074703bbe70e0acb7ab92d40f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 03e42650b7af55aca5d0eacb803cc58b30f70ebc4148dc8226e39242c266b061824d1a38f3291b5e5b2a8961d216703d5d03fd2ea31faef4027ddc4b1e097d81
|
7
|
+
data.tar.gz: 849ebc1a05f62eb3676d7dfc6550b975f15da62172cdaba86339ad3c8974cf9610c889f72084de908d8739598ff5447b1f5dcaf1e595ede9af6c32c0802685a2
|
data/.rubocop.yml
CHANGED
@@ -31,6 +31,9 @@ Style/FormatStringToken:
|
|
31
31
|
EnforcedStyle: template
|
32
32
|
Style/AsciiComments:
|
33
33
|
Enabled: false
|
34
|
+
Style/FrozenStringLiteralComment:
|
35
|
+
Exclude:
|
36
|
+
- "gemfiles/*"
|
34
37
|
|
35
38
|
Metrics/BlockLength:
|
36
39
|
Exclude:
|
@@ -45,4 +48,4 @@ Lint/UselessSetterCall:
|
|
45
48
|
Exclude:
|
46
49
|
# Rubocop is incorrectly flagging `term_on_empty` as local
|
47
50
|
# See: https://github.com/bbatsov/rubocop/issues/5420
|
48
|
-
- lib/resque/kubernetes/
|
51
|
+
- lib/resque/kubernetes/manifest_conformance.rb
|
data/CHANGELOG.md
CHANGED
data/README.md
CHANGED
data/lib/resque/kubernetes.rb
CHANGED
@@ -8,6 +8,7 @@ require "resque/kubernetes/deep_hash"
|
|
8
8
|
require "resque/kubernetes/dns_safe_random"
|
9
9
|
require "resque/kubernetes/job"
|
10
10
|
require "resque/kubernetes/jobs_manager"
|
11
|
+
require "resque/kubernetes/manifest_conformance"
|
11
12
|
require "resque/kubernetes/version"
|
12
13
|
|
13
14
|
module Resque
|
@@ -12,13 +12,13 @@ module Resque
|
|
12
12
|
class << self
|
13
13
|
# Returns an n-length string of DNS-safe characters.
|
14
14
|
#
|
15
|
-
#
|
16
|
-
def random_chars(
|
17
|
-
s = [SecureRandom.random_bytes(
|
15
|
+
# number: The number of characters to return (default 5).
|
16
|
+
def random_chars(number = 5)
|
17
|
+
s = [SecureRandom.random_bytes(number)].pack("m*")
|
18
18
|
s.delete!("=\n")
|
19
19
|
s.tr!("+/_-", "0")
|
20
20
|
s.tr!("A-Z", "a-z")
|
21
|
-
s[0...
|
21
|
+
s[0...number]
|
22
22
|
end
|
23
23
|
end
|
24
24
|
end
|
@@ -82,13 +82,14 @@ module Resque
|
|
82
82
|
end
|
83
83
|
|
84
84
|
# A before_enqueue hook that adds worker jobs to the cluster.
|
85
|
-
def before_enqueue_kubernetes_job(*
|
85
|
+
def before_enqueue_kubernetes_job(*_args)
|
86
86
|
if defined? Rails
|
87
87
|
return unless Resque::Kubernetes.environments.include?(Rails.env)
|
88
88
|
end
|
89
89
|
|
90
90
|
manager = JobsManager.new(self)
|
91
91
|
manager.reap_finished_jobs
|
92
|
+
manager.reap_finished_pods
|
92
93
|
manager.apply_kubernetes_job
|
93
94
|
end
|
94
95
|
|
@@ -2,10 +2,14 @@
|
|
2
2
|
|
3
3
|
require "kubeclient"
|
4
4
|
|
5
|
+
require_relative "manifest_conformance"
|
6
|
+
|
5
7
|
module Resque
|
6
8
|
module Kubernetes
|
7
9
|
# Spins up Kubernetes Jobs to run Resque workers.
|
8
10
|
class JobsManager
|
11
|
+
include Resque::Kubernetes::ManifestConformance
|
12
|
+
|
9
13
|
attr_reader :owner
|
10
14
|
private :owner
|
11
15
|
|
@@ -24,6 +28,16 @@ module Resque
|
|
24
28
|
end
|
25
29
|
end
|
26
30
|
|
31
|
+
def reap_finished_pods
|
32
|
+
finished_pods.each do |pod|
|
33
|
+
begin
|
34
|
+
pods_client.delete_pod(pod.metadata.name, pod.metadata.namespace)
|
35
|
+
rescue KubeException => e
|
36
|
+
raise unless e.error_code == 404
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
27
41
|
def apply_kubernetes_job
|
28
42
|
manifest = DeepHash.new.merge!(owner.job_manifest)
|
29
43
|
ensure_namespace(manifest)
|
@@ -43,6 +57,10 @@ module Resque
|
|
43
57
|
@jobs_client ||= client("/apis/batch")
|
44
58
|
end
|
45
59
|
|
60
|
+
def pods_client
|
61
|
+
@pods_client ||= client("")
|
62
|
+
end
|
63
|
+
|
46
64
|
def client(scope)
|
47
65
|
return Resque::Kubernetes.kubeclient if Resque::Kubernetes.kubeclient
|
48
66
|
|
@@ -58,6 +76,15 @@ module Resque
|
|
58
76
|
resque_jobs.select { |job| job.spec.completions == job.status.succeeded }
|
59
77
|
end
|
60
78
|
|
79
|
+
def finished_pods
|
80
|
+
resque_jobs = pods_client.get_pods(label_selector: "resque-kubernetes=pod")
|
81
|
+
resque_jobs.select do |pod|
|
82
|
+
pod.status.phase == "Succeeded" && pod.status.containerStatuses.all? do |status|
|
83
|
+
status.state.terminated.reason == "Completed"
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
61
88
|
def jobs_maxed?(name, namespace)
|
62
89
|
resque_jobs = jobs_client.get_jobs(
|
63
90
|
label_selector: "resque-kubernetes=job,resque-kubernetes-group=#{name}",
|
@@ -66,49 +93,6 @@ module Resque
|
|
66
93
|
running = resque_jobs.reject { |job| job.spec.completions == job.status.succeeded }
|
67
94
|
running.size >= owner.max_workers
|
68
95
|
end
|
69
|
-
|
70
|
-
def adjust_manifest(manifest)
|
71
|
-
add_labels(manifest)
|
72
|
-
ensure_term_on_empty(manifest)
|
73
|
-
ensure_reset_policy(manifest)
|
74
|
-
update_job_name(manifest)
|
75
|
-
end
|
76
|
-
|
77
|
-
def add_labels(manifest)
|
78
|
-
manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
|
79
|
-
manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
|
80
|
-
manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
|
81
|
-
end
|
82
|
-
|
83
|
-
def ensure_term_on_empty(manifest)
|
84
|
-
manifest["spec"]["template"]["spec"] ||= {}
|
85
|
-
manifest["spec"]["template"]["spec"]["containers"] ||= []
|
86
|
-
manifest["spec"]["template"]["spec"]["containers"].each do |container|
|
87
|
-
container_term_on_empty(container)
|
88
|
-
end
|
89
|
-
end
|
90
|
-
|
91
|
-
def container_term_on_empty(container)
|
92
|
-
container["env"] ||= []
|
93
|
-
term_on_empty = container["env"].find { |env| env["name"] == "INTERVAL" }
|
94
|
-
unless term_on_empty
|
95
|
-
term_on_empty = {"name" => "INTERVAL"}
|
96
|
-
container["env"] << term_on_empty
|
97
|
-
end
|
98
|
-
term_on_empty["value"] = "0"
|
99
|
-
end
|
100
|
-
|
101
|
-
def ensure_reset_policy(manifest)
|
102
|
-
manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
|
103
|
-
end
|
104
|
-
|
105
|
-
def ensure_namespace(manifest)
|
106
|
-
manifest["metadata"]["namespace"] ||= @default_namespace
|
107
|
-
end
|
108
|
-
|
109
|
-
def update_job_name(manifest)
|
110
|
-
manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
|
111
|
-
end
|
112
96
|
end
|
113
97
|
end
|
114
98
|
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Resque
|
4
|
+
module Kubernetes
|
5
|
+
# Provides methods to ensure a mannifest conforms to a job specification
|
6
|
+
# and includes details needed for resque-kubernetes
|
7
|
+
module ManifestConformance
|
8
|
+
def adjust_manifest(manifest)
|
9
|
+
add_labels(manifest)
|
10
|
+
ensure_term_on_empty(manifest)
|
11
|
+
ensure_reset_policy(manifest)
|
12
|
+
update_job_name(manifest)
|
13
|
+
end
|
14
|
+
|
15
|
+
def add_labels(manifest)
|
16
|
+
manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
|
17
|
+
manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
|
18
|
+
manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
|
19
|
+
end
|
20
|
+
|
21
|
+
def ensure_term_on_empty(manifest)
|
22
|
+
manifest["spec"]["template"]["spec"] ||= {}
|
23
|
+
manifest["spec"]["template"]["spec"]["containers"] ||= []
|
24
|
+
manifest["spec"]["template"]["spec"]["containers"].each do |container|
|
25
|
+
container_term_on_empty(container)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def container_term_on_empty(container)
|
30
|
+
container["env"] ||= []
|
31
|
+
term_on_empty = container["env"].find { |env| env["name"] == "INTERVAL" }
|
32
|
+
unless term_on_empty
|
33
|
+
term_on_empty = {"name" => "INTERVAL"}
|
34
|
+
container["env"] << term_on_empty
|
35
|
+
end
|
36
|
+
term_on_empty["value"] = "0"
|
37
|
+
end
|
38
|
+
|
39
|
+
def ensure_reset_policy(manifest)
|
40
|
+
manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
|
41
|
+
end
|
42
|
+
|
43
|
+
def ensure_namespace(manifest)
|
44
|
+
manifest["metadata"]["namespace"] ||= @default_namespace
|
45
|
+
end
|
46
|
+
|
47
|
+
def update_job_name(manifest)
|
48
|
+
manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
data/resque-kubernetes.gemspec
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
lib = File.expand_path("
|
3
|
+
lib = File.expand_path("lib", __dir__)
|
4
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
5
5
|
require "resque/kubernetes/version"
|
6
6
|
|
@@ -29,7 +29,8 @@ Gem::Specification.new do |spec|
|
|
29
29
|
spec.add_development_dependency "googleauth", "~> 0.6"
|
30
30
|
spec.add_development_dependency "rake", "~> 12.3"
|
31
31
|
spec.add_development_dependency "rspec", "~> 3.7"
|
32
|
-
|
32
|
+
# Pinned less than 0.56.0 until this is resolved: https://github.com/rubocop-hq/rubocop/issues/5975
|
33
|
+
spec.add_development_dependency "rubocop", "~> 0.52", ">= 0.52.1", "< 0.56.0"
|
33
34
|
|
34
35
|
spec.add_dependency "kubeclient", ">= 3.1.2", "< 5.0"
|
35
36
|
spec.add_dependency "resque", "~> 1.26"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: resque-kubernetes
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeremy Wadsack
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-11-13 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: appraisal
|
@@ -104,6 +104,9 @@ dependencies:
|
|
104
104
|
- - ">="
|
105
105
|
- !ruby/object:Gem::Version
|
106
106
|
version: 0.52.1
|
107
|
+
- - "<"
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: 0.56.0
|
107
110
|
type: :development
|
108
111
|
prerelease: false
|
109
112
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -114,6 +117,9 @@ dependencies:
|
|
114
117
|
- - ">="
|
115
118
|
- !ruby/object:Gem::Version
|
116
119
|
version: 0.52.1
|
120
|
+
- - "<"
|
121
|
+
- !ruby/object:Gem::Version
|
122
|
+
version: 0.56.0
|
117
123
|
- !ruby/object:Gem::Dependency
|
118
124
|
name: kubeclient
|
119
125
|
requirement: !ruby/object:Gem::Requirement
|
@@ -180,6 +186,7 @@ files:
|
|
180
186
|
- lib/resque/kubernetes/dns_safe_random.rb
|
181
187
|
- lib/resque/kubernetes/job.rb
|
182
188
|
- lib/resque/kubernetes/jobs_manager.rb
|
189
|
+
- lib/resque/kubernetes/manifest_conformance.rb
|
183
190
|
- lib/resque/kubernetes/version.rb
|
184
191
|
- resque-kubernetes.gemspec
|
185
192
|
homepage: https://github.com/keylimetoolbox/resque-kubernetes
|