resque-kubernetes 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: dd653a29a38d097f457584f8bb8a4b305dc14f1f
4
- data.tar.gz: 4ac673bc617fdd91c99fe3b51f0c943f214655f6
3
+ metadata.gz: bbfbc8c610d079165438b6a635043070730ecc0a
4
+ data.tar.gz: a409a4638079687074703bbe70e0acb7ab92d40f
5
5
  SHA512:
6
- metadata.gz: 142a38eea17c227b6339eef837b58e4835d3fdf05d7f76f85f30cde00c25f6d20b3ee1fa59795b030549283bfcae28eb49da8159449bd3004066c258c5a7b4d0
7
- data.tar.gz: 707c7a76de380e87d8da7c4f4651d924f9b7992d5bdedcf48150a0ce8c225cda21d9f252f4a4b907dae373c5c012be2f9ec449bb69d90501a711607f2161b3e2
6
+ metadata.gz: 03e42650b7af55aca5d0eacb803cc58b30f70ebc4148dc8226e39242c266b061824d1a38f3291b5e5b2a8961d216703d5d03fd2ea31faef4027ddc4b1e097d81
7
+ data.tar.gz: 849ebc1a05f62eb3676d7dfc6550b975f15da62172cdaba86339ad3c8974cf9610c889f72084de908d8739598ff5447b1f5dcaf1e595ede9af6c32c0802685a2
@@ -31,6 +31,9 @@ Style/FormatStringToken:
31
31
  EnforcedStyle: template
32
32
  Style/AsciiComments:
33
33
  Enabled: false
34
+ Style/FrozenStringLiteralComment:
35
+ Exclude:
36
+ - "gemfiles/*"
34
37
 
35
38
  Metrics/BlockLength:
36
39
  Exclude:
@@ -45,4 +48,4 @@ Lint/UselessSetterCall:
45
48
  Exclude:
46
49
  # Rubocop is incorrectly flagging `term_on_empty` as local
47
50
  # See: https://github.com/bbatsov/rubocop/issues/5420
48
- - lib/resque/kubernetes/jobs_manager.rb
51
+ - lib/resque/kubernetes/manifest_conformance.rb
@@ -1,3 +1,6 @@
1
+ # v1.2.0
2
+ - Clean up finished pods that have successfully completed
3
+
1
4
  # v1.1.0
2
5
  - Fix design to set `INTERVAL=0` for the worker in the Kubernetes
3
6
  job manifest, which tells `resque` to work until there are no more jobs,
data/README.md CHANGED
@@ -1,4 +1,5 @@
1
1
  # Resque::Kubernetes
2
+ [![Gem Version](https://badge.fury.io/rb/resque-kubernetes.svg)](https://badge.fury.io/rb/resque-kubernetes)
2
3
 
3
4
  Run Resque (and ActiveJob) Workers as Kubernetes Jobs!
4
5
 
@@ -8,6 +8,7 @@ require "resque/kubernetes/deep_hash"
8
8
  require "resque/kubernetes/dns_safe_random"
9
9
  require "resque/kubernetes/job"
10
10
  require "resque/kubernetes/jobs_manager"
11
+ require "resque/kubernetes/manifest_conformance"
11
12
  require "resque/kubernetes/version"
12
13
 
13
14
  module Resque
@@ -12,13 +12,13 @@ module Resque
12
12
  class << self
13
13
  # Returns an n-length string of DNS-safe characters.
14
14
  #
15
- # n: The number of characters to return (default 5).
16
- def random_chars(n = 5)
17
- s = [SecureRandom.random_bytes(n)].pack("m*")
15
+ # number: The number of characters to return (default 5).
16
+ def random_chars(number = 5)
17
+ s = [SecureRandom.random_bytes(number)].pack("m*")
18
18
  s.delete!("=\n")
19
19
  s.tr!("+/_-", "0")
20
20
  s.tr!("A-Z", "a-z")
21
- s[0...n]
21
+ s[0...number]
22
22
  end
23
23
  end
24
24
  end
@@ -82,13 +82,14 @@ module Resque
82
82
  end
83
83
 
84
84
  # A before_enqueue hook that adds worker jobs to the cluster.
85
- def before_enqueue_kubernetes_job(*_)
85
+ def before_enqueue_kubernetes_job(*_args)
86
86
  if defined? Rails
87
87
  return unless Resque::Kubernetes.environments.include?(Rails.env)
88
88
  end
89
89
 
90
90
  manager = JobsManager.new(self)
91
91
  manager.reap_finished_jobs
92
+ manager.reap_finished_pods
92
93
  manager.apply_kubernetes_job
93
94
  end
94
95
 
@@ -2,10 +2,14 @@
2
2
 
3
3
  require "kubeclient"
4
4
 
5
+ require_relative "manifest_conformance"
6
+
5
7
  module Resque
6
8
  module Kubernetes
7
9
  # Spins up Kubernetes Jobs to run Resque workers.
8
10
  class JobsManager
11
+ include Resque::Kubernetes::ManifestConformance
12
+
9
13
  attr_reader :owner
10
14
  private :owner
11
15
 
@@ -24,6 +28,16 @@ module Resque
24
28
  end
25
29
  end
26
30
 
31
+ def reap_finished_pods
32
+ finished_pods.each do |pod|
33
+ begin
34
+ pods_client.delete_pod(pod.metadata.name, pod.metadata.namespace)
35
+ rescue KubeException => e
36
+ raise unless e.error_code == 404
37
+ end
38
+ end
39
+ end
40
+
27
41
  def apply_kubernetes_job
28
42
  manifest = DeepHash.new.merge!(owner.job_manifest)
29
43
  ensure_namespace(manifest)
@@ -43,6 +57,10 @@ module Resque
43
57
  @jobs_client ||= client("/apis/batch")
44
58
  end
45
59
 
60
+ def pods_client
61
+ @pods_client ||= client("")
62
+ end
63
+
46
64
  def client(scope)
47
65
  return Resque::Kubernetes.kubeclient if Resque::Kubernetes.kubeclient
48
66
 
@@ -58,6 +76,15 @@ module Resque
58
76
  resque_jobs.select { |job| job.spec.completions == job.status.succeeded }
59
77
  end
60
78
 
79
+ def finished_pods
80
+ resque_jobs = pods_client.get_pods(label_selector: "resque-kubernetes=pod")
81
+ resque_jobs.select do |pod|
82
+ pod.status.phase == "Succeeded" && pod.status.containerStatuses.all? do |status|
83
+ status.state.terminated.reason == "Completed"
84
+ end
85
+ end
86
+ end
87
+
61
88
  def jobs_maxed?(name, namespace)
62
89
  resque_jobs = jobs_client.get_jobs(
63
90
  label_selector: "resque-kubernetes=job,resque-kubernetes-group=#{name}",
@@ -66,49 +93,6 @@ module Resque
66
93
  running = resque_jobs.reject { |job| job.spec.completions == job.status.succeeded }
67
94
  running.size >= owner.max_workers
68
95
  end
69
-
70
- def adjust_manifest(manifest)
71
- add_labels(manifest)
72
- ensure_term_on_empty(manifest)
73
- ensure_reset_policy(manifest)
74
- update_job_name(manifest)
75
- end
76
-
77
- def add_labels(manifest)
78
- manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
79
- manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
80
- manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
81
- end
82
-
83
- def ensure_term_on_empty(manifest)
84
- manifest["spec"]["template"]["spec"] ||= {}
85
- manifest["spec"]["template"]["spec"]["containers"] ||= []
86
- manifest["spec"]["template"]["spec"]["containers"].each do |container|
87
- container_term_on_empty(container)
88
- end
89
- end
90
-
91
- def container_term_on_empty(container)
92
- container["env"] ||= []
93
- term_on_empty = container["env"].find { |env| env["name"] == "INTERVAL" }
94
- unless term_on_empty
95
- term_on_empty = {"name" => "INTERVAL"}
96
- container["env"] << term_on_empty
97
- end
98
- term_on_empty["value"] = "0"
99
- end
100
-
101
- def ensure_reset_policy(manifest)
102
- manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
103
- end
104
-
105
- def ensure_namespace(manifest)
106
- manifest["metadata"]["namespace"] ||= @default_namespace
107
- end
108
-
109
- def update_job_name(manifest)
110
- manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
111
- end
112
96
  end
113
97
  end
114
98
  end
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Resque
4
+ module Kubernetes
5
+ # Provides methods to ensure a mannifest conforms to a job specification
6
+ # and includes details needed for resque-kubernetes
7
+ module ManifestConformance
8
+ def adjust_manifest(manifest)
9
+ add_labels(manifest)
10
+ ensure_term_on_empty(manifest)
11
+ ensure_reset_policy(manifest)
12
+ update_job_name(manifest)
13
+ end
14
+
15
+ def add_labels(manifest)
16
+ manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
17
+ manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
18
+ manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
19
+ end
20
+
21
+ def ensure_term_on_empty(manifest)
22
+ manifest["spec"]["template"]["spec"] ||= {}
23
+ manifest["spec"]["template"]["spec"]["containers"] ||= []
24
+ manifest["spec"]["template"]["spec"]["containers"].each do |container|
25
+ container_term_on_empty(container)
26
+ end
27
+ end
28
+
29
+ def container_term_on_empty(container)
30
+ container["env"] ||= []
31
+ term_on_empty = container["env"].find { |env| env["name"] == "INTERVAL" }
32
+ unless term_on_empty
33
+ term_on_empty = {"name" => "INTERVAL"}
34
+ container["env"] << term_on_empty
35
+ end
36
+ term_on_empty["value"] = "0"
37
+ end
38
+
39
+ def ensure_reset_policy(manifest)
40
+ manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
41
+ end
42
+
43
+ def ensure_namespace(manifest)
44
+ manifest["metadata"]["namespace"] ||= @default_namespace
45
+ end
46
+
47
+ def update_job_name(manifest)
48
+ manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
49
+ end
50
+ end
51
+ end
52
+ end
@@ -2,6 +2,6 @@
2
2
 
3
3
  module Resque
4
4
  module Kubernetes
5
- VERSION = "1.1.0"
5
+ VERSION = "1.2.0"
6
6
  end
7
7
  end
@@ -1,6 +1,6 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- lib = File.expand_path("../lib", __FILE__)
3
+ lib = File.expand_path("lib", __dir__)
4
4
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
5
  require "resque/kubernetes/version"
6
6
 
@@ -29,7 +29,8 @@ Gem::Specification.new do |spec|
29
29
  spec.add_development_dependency "googleauth", "~> 0.6"
30
30
  spec.add_development_dependency "rake", "~> 12.3"
31
31
  spec.add_development_dependency "rspec", "~> 3.7"
32
- spec.add_development_dependency "rubocop", "~> 0.52", ">= 0.52.1"
32
+ # Pinned less than 0.56.0 until this is resolved: https://github.com/rubocop-hq/rubocop/issues/5975
33
+ spec.add_development_dependency "rubocop", "~> 0.52", ">= 0.52.1", "< 0.56.0"
33
34
 
34
35
  spec.add_dependency "kubeclient", ">= 3.1.2", "< 5.0"
35
36
  spec.add_dependency "resque", "~> 1.26"
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: resque-kubernetes
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Jeremy Wadsack
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-09-27 00:00:00.000000000 Z
11
+ date: 2018-11-13 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: appraisal
@@ -104,6 +104,9 @@ dependencies:
104
104
  - - ">="
105
105
  - !ruby/object:Gem::Version
106
106
  version: 0.52.1
107
+ - - "<"
108
+ - !ruby/object:Gem::Version
109
+ version: 0.56.0
107
110
  type: :development
108
111
  prerelease: false
109
112
  version_requirements: !ruby/object:Gem::Requirement
@@ -114,6 +117,9 @@ dependencies:
114
117
  - - ">="
115
118
  - !ruby/object:Gem::Version
116
119
  version: 0.52.1
120
+ - - "<"
121
+ - !ruby/object:Gem::Version
122
+ version: 0.56.0
117
123
  - !ruby/object:Gem::Dependency
118
124
  name: kubeclient
119
125
  requirement: !ruby/object:Gem::Requirement
@@ -180,6 +186,7 @@ files:
180
186
  - lib/resque/kubernetes/dns_safe_random.rb
181
187
  - lib/resque/kubernetes/job.rb
182
188
  - lib/resque/kubernetes/jobs_manager.rb
189
+ - lib/resque/kubernetes/manifest_conformance.rb
183
190
  - lib/resque/kubernetes/version.rb
184
191
  - resque-kubernetes.gemspec
185
192
  homepage: https://github.com/keylimetoolbox/resque-kubernetes