resque-kubernetes 0.4.0 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +48 -0
- data/CHANGELOG.md +17 -0
- data/Gemfile +3 -1
- data/README.md +26 -0
- data/Rakefile +11 -1
- data/bin/console +1 -0
- data/lib/resque/kubernetes.rb +6 -0
- data/lib/resque/kubernetes/configurable.rb +9 -5
- data/lib/resque/kubernetes/context_factory.rb +49 -0
- data/lib/resque/kubernetes/deep_hash.rb +32 -0
- data/lib/resque/kubernetes/dns_safe_random.rb +26 -0
- data/lib/resque/kubernetes/job.rb +110 -85
- data/lib/resque/kubernetes/version.rb +3 -1
- data/lib/resque/kubernetes/worker.rb +14 -3
- data/resque-kubernetes.gemspec +13 -9
- metadata +51 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5f22de9195620c7f00040e39027aebaefccb4e52
|
4
|
+
data.tar.gz: 524587419d578689b3719c16bf21105a26fdfa03
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 428eba2f1ba7707ccc9992a684530b50b83f280b824b297984b583ac068d3fdf738bcd8edd5806ac49865d9666fea41cd4f6a7a255790955d0c48c2d8a232113
|
7
|
+
data.tar.gz: e87c50ee0bd9742e5495707069bbf97dd90a7f080bf78a5865a39ef912f6f59889d5feddddce309f5469ebfe67c9955c9909561714fe940d71994f0f3c6ed869
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
Documentation:
|
2
|
+
Exclude:
|
3
|
+
- "**/railtie.rb"
|
4
|
+
- "spec/**/*"
|
5
|
+
|
6
|
+
Style/StringLiterals:
|
7
|
+
EnforcedStyle: double_quotes
|
8
|
+
Metrics/LineLength:
|
9
|
+
Max: 120
|
10
|
+
Layout/AlignHash:
|
11
|
+
EnforcedHashRocketStyle: table
|
12
|
+
EnforcedColonStyle: table
|
13
|
+
Layout/SpaceInsideHashLiteralBraces:
|
14
|
+
EnforcedStyle: no_space
|
15
|
+
Style/RaiseArgs:
|
16
|
+
EnforcedStyle: compact
|
17
|
+
Style/EmptyMethod:
|
18
|
+
EnforcedStyle: expanded
|
19
|
+
Layout/IndentArray:
|
20
|
+
IndentationWidth: 4
|
21
|
+
Layout/IndentHash:
|
22
|
+
IndentationWidth: 4
|
23
|
+
Style/ConditionalAssignment:
|
24
|
+
EnforcedStyle: assign_inside_condition
|
25
|
+
Layout/FirstParameterIndentation:
|
26
|
+
IndentationWidth: 4
|
27
|
+
Layout/MultilineOperationIndentation:
|
28
|
+
IndentationWidth: 4
|
29
|
+
EnforcedStyle: indented
|
30
|
+
Style/FormatStringToken:
|
31
|
+
EnforcedStyle: template
|
32
|
+
Style/AsciiComments:
|
33
|
+
Enabled: false
|
34
|
+
|
35
|
+
Metrics/BlockLength:
|
36
|
+
Exclude:
|
37
|
+
- "keylime-service-api.gemspec"
|
38
|
+
- "spec/**/*"
|
39
|
+
|
40
|
+
Layout/EmptyLinesAroundBlockBody:
|
41
|
+
Exclude:
|
42
|
+
- "spec/**/*"
|
43
|
+
|
44
|
+
Lint/UselessSetterCall:
|
45
|
+
Exclude:
|
46
|
+
# Rubocop is incorrectly flagging `term_on_empty` as local
|
47
|
+
# See: https://github.com/bbatsov/rubocop/issues/5420
|
48
|
+
- lib/resque/kubernetes/job.rb
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
# 0.5.0
|
2
|
+
- Maximum workers can no be configured per job type
|
3
|
+
- Fix a crash when cleaning up a job that was removed by another process
|
4
|
+
- No longer clean up pods because cleaning up finished jobs takes care of that
|
5
|
+
- Apply rubocop, and bundler-audit rules
|
6
|
+
|
7
|
+
# 0.4.0
|
8
|
+
- Syntax error fix
|
9
|
+
|
10
|
+
# 0.3.0
|
11
|
+
- Syntax error fix
|
12
|
+
|
13
|
+
# 0.2.0
|
14
|
+
- Fix for running in GKE cluster and production Rails environment
|
15
|
+
|
16
|
+
# 0.1.0
|
17
|
+
- Initial release
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -51,6 +51,8 @@ class ResourceIntensiveJob
|
|
51
51
|
|
52
52
|
def job_manifest
|
53
53
|
<<-EOD
|
54
|
+
apiVersion: batch/v1
|
55
|
+
kind: Job
|
54
56
|
metadata:
|
55
57
|
name: worker-job
|
56
58
|
spec:
|
@@ -113,6 +115,30 @@ it.
|
|
113
115
|
|
114
116
|
If you don't want more than one job running at a time then set this to 1.
|
115
117
|
|
118
|
+
Beyond this global scope you can adjust the total number of workers on each
|
119
|
+
individual Resque Job type by overriding the `max_workers` class method for the job.
|
120
|
+
If you change this, the value returned by that method takes precedence over the
|
121
|
+
global value.
|
122
|
+
|
123
|
+
```ruby
|
124
|
+
class ResourceIntensiveJob
|
125
|
+
extend Resque::Kubernetes::Job
|
126
|
+
|
127
|
+
def perform
|
128
|
+
# ...
|
129
|
+
end
|
130
|
+
|
131
|
+
def job_manifest
|
132
|
+
# ...
|
133
|
+
end
|
134
|
+
|
135
|
+
def max_workers
|
136
|
+
# Simply return an integer value, or do something more complicated if needed.
|
137
|
+
105
|
138
|
+
end
|
139
|
+
end
|
140
|
+
```
|
141
|
+
|
116
142
|
## To Do
|
117
143
|
|
118
144
|
- We probably need better namespace support, particularly for reaping
|
data/Rakefile
CHANGED
@@ -1,6 +1,16 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "bundler/gem_tasks"
|
2
4
|
require "rspec/core/rake_task"
|
5
|
+
require "rubocop/rake_task"
|
6
|
+
require "bundler/audit/task"
|
3
7
|
|
4
8
|
RSpec::Core::RakeTask.new(:spec)
|
9
|
+
RuboCop::RakeTask.new
|
10
|
+
Bundler::Audit::Task.new
|
11
|
+
|
12
|
+
# Remove default and replace with a series of test tasks
|
13
|
+
task default: []
|
14
|
+
Rake::Task[:default].clear
|
5
15
|
|
6
|
-
task :
|
16
|
+
task default: %w[spec rubocop bundle:audit]
|
data/bin/console
CHANGED
data/lib/resque/kubernetes.rb
CHANGED
@@ -1,9 +1,15 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "resque/kubernetes/context_factory"
|
4
|
+
require "resque/kubernetes/deep_hash"
|
5
|
+
require "resque/kubernetes/dns_safe_random"
|
1
6
|
require "resque/kubernetes/job"
|
2
7
|
require "resque/kubernetes/version"
|
3
8
|
require "resque/kubernetes/worker"
|
4
9
|
require "resque/kubernetes/configurable"
|
5
10
|
|
6
11
|
module Resque
|
12
|
+
# Run Resque Jobs as Kubernetes Jobs with autoscaling.
|
7
13
|
module Kubernetes
|
8
14
|
extend Configurable
|
9
15
|
|
@@ -1,11 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Resque
|
2
4
|
module Kubernetes
|
5
|
+
# Provides configuration settings, with default values, for the gem.
|
3
6
|
module Configurable
|
4
|
-
|
5
7
|
def configuration
|
6
8
|
yield self
|
7
9
|
end
|
8
10
|
|
11
|
+
# Define a configuration setting and its default value.
|
12
|
+
#
|
13
|
+
# name: The name of the setting.
|
14
|
+
# default: A default value for the setting. (Optional)
|
9
15
|
def define_setting(name, default = nil)
|
10
16
|
class_variable_set("@@#{name}", default)
|
11
17
|
|
@@ -14,7 +20,7 @@ module Resque
|
|
14
20
|
end
|
15
21
|
|
16
22
|
define_class_method name do
|
17
|
-
|
23
|
+
class_variable_get("@@#{name}")
|
18
24
|
end
|
19
25
|
end
|
20
26
|
|
@@ -25,8 +31,6 @@ module Resque
|
|
25
31
|
define_method(name, &block)
|
26
32
|
end
|
27
33
|
end
|
28
|
-
|
29
|
-
|
30
|
-
end
|
34
|
+
end
|
31
35
|
end
|
32
36
|
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "kubeclient"
|
4
|
+
|
5
|
+
module Resque
|
6
|
+
module Kubernetes
|
7
|
+
# Create a context for `Kubeclient` depending on the environment.
|
8
|
+
class ContextFactory
|
9
|
+
class << self
|
10
|
+
def context
|
11
|
+
# TODO: Add ability to load this from config
|
12
|
+
|
13
|
+
if File.exist?("/var/run/secrets/kubernetes.io/serviceaccount/token")
|
14
|
+
# When running in GKE/k8s cluster, use the service account secret token and ca bundle
|
15
|
+
well_known_context
|
16
|
+
elsif File.exist?(kubeconfig)
|
17
|
+
# When running in development, use the config file for `kubectl` and default application credentials
|
18
|
+
kubectl_context
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def well_known_context
|
25
|
+
Kubeclient::Config::Context.new(
|
26
|
+
"https://kubernetes",
|
27
|
+
"v1",
|
28
|
+
{ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"},
|
29
|
+
bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
30
|
+
)
|
31
|
+
end
|
32
|
+
|
33
|
+
def kubectl_context
|
34
|
+
config = Kubeclient::Config.read(kubeconfig)
|
35
|
+
Kubeclient::Config::Context.new(
|
36
|
+
config.context.api_endpoint,
|
37
|
+
config.context.api_version,
|
38
|
+
config.context.ssl_options,
|
39
|
+
use_default_gcp: true
|
40
|
+
)
|
41
|
+
end
|
42
|
+
|
43
|
+
def kubeconfig
|
44
|
+
File.join(ENV["HOME"], ".kube", "config")
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Resque
|
4
|
+
module Kubernetes
|
5
|
+
# A subclass of Hash that allows nested keys to be added
|
6
|
+
# and ensures the interim hashes exist.
|
7
|
+
class DeepHash < Hash
|
8
|
+
# Add keys and a value to nested hashes.
|
9
|
+
#
|
10
|
+
# keys: An array of keys.
|
11
|
+
# value: A value to assign to the key in the ultimate hash.
|
12
|
+
#
|
13
|
+
# Example:
|
14
|
+
# h = DeepHash.new
|
15
|
+
# h.deep_add(%w[one two three], "deep")
|
16
|
+
# deep = h["one"]["two"]["three"]
|
17
|
+
def deep_add(keys, value)
|
18
|
+
last_key = keys.pop
|
19
|
+
|
20
|
+
m = self
|
21
|
+
keys.each do |key|
|
22
|
+
m[key] ||= {}
|
23
|
+
m = m[key]
|
24
|
+
end
|
25
|
+
|
26
|
+
m[last_key] = value
|
27
|
+
|
28
|
+
m
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "securerandom"
|
4
|
+
|
5
|
+
module Resque
|
6
|
+
module Kubernetes
|
7
|
+
# Simple utility to generate a string of DNS-safe characters.
|
8
|
+
#
|
9
|
+
# Example:
|
10
|
+
# str = DNSSafeRandom.random_characters
|
11
|
+
class DNSSafeRandom
|
12
|
+
class << self
|
13
|
+
# Returns an n-length string of DNS-safe characters.
|
14
|
+
#
|
15
|
+
# n: The number of characters to return (default 5).
|
16
|
+
def random_chars(n = 5)
|
17
|
+
s = [SecureRandom.random_bytes(n)].pack("m*")
|
18
|
+
s.delete!("=\n")
|
19
|
+
s.tr!("+/_-", "0")
|
20
|
+
s.tr!("A-Z", "a-z")
|
21
|
+
s[0...n]
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -1,20 +1,80 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require "kubeclient"
|
3
4
|
|
4
5
|
module Resque
|
5
6
|
module Kubernetes
|
7
|
+
# Resque hook to autoscale Kubernetes Jobs for workers.
|
8
|
+
#
|
9
|
+
# To use, extend your Resque job class with this module and then define a
|
10
|
+
# class method `job_manifest` that produces the Kubernetes Job manifest.
|
11
|
+
#
|
12
|
+
# Example:
|
13
|
+
#
|
14
|
+
# class ResourceIntensiveJob
|
15
|
+
# extend Resque::Kubernetes::Job
|
16
|
+
#
|
17
|
+
# def perform
|
18
|
+
# # ... your existing code
|
19
|
+
# end
|
20
|
+
#
|
21
|
+
# def job_manifest
|
22
|
+
# <<-EOD
|
23
|
+
# apiVersion: batch/v1
|
24
|
+
# kind: Job
|
25
|
+
# metadata:
|
26
|
+
# name: worker-job
|
27
|
+
# spec:
|
28
|
+
# template:
|
29
|
+
# metadata:
|
30
|
+
# name: worker-job
|
31
|
+
# spec:
|
32
|
+
# containers:
|
33
|
+
# - name: worker
|
34
|
+
# image: us.gcr.io/project-id/some-resque-worker
|
35
|
+
# env:
|
36
|
+
# - name: QUEUE
|
37
|
+
# value: high-memory
|
38
|
+
# EOD
|
39
|
+
# end
|
40
|
+
# end
|
6
41
|
module Job
|
7
|
-
|
42
|
+
# A before_enqueue hook that adds worker jobs to the cluster.
|
8
43
|
def before_enqueue_kubernetes_job(*_)
|
9
44
|
if defined? Rails
|
10
45
|
return unless Resque::Kubernetes.environments.include?(Rails.env)
|
11
46
|
end
|
12
47
|
|
13
48
|
reap_finished_jobs
|
14
|
-
reap_finished_pods
|
15
49
|
apply_kubernetes_job
|
16
50
|
end
|
17
51
|
|
52
|
+
protected
|
53
|
+
|
54
|
+
# Return the maximum number of workers to autoscale the job to.
|
55
|
+
#
|
56
|
+
# While the number of active Kubernetes Jobs is less than this number,
|
57
|
+
# the gem will add new Jobs to auto-scale the workers.
|
58
|
+
#
|
59
|
+
# By default, this returns `Resque::Kubernetes.max_workers` from the gem
|
60
|
+
# configuration. You may override this method to return any other value,
|
61
|
+
# either as a simple integer or with some complex logic.
|
62
|
+
#
|
63
|
+
# Example:
|
64
|
+
# def max_workers
|
65
|
+
# # A simple integer
|
66
|
+
# 105
|
67
|
+
# end
|
68
|
+
#
|
69
|
+
# Example:
|
70
|
+
# def max_workers
|
71
|
+
# # Scale based on time of day
|
72
|
+
# Time.now.hour < 8 ? 15 : 5
|
73
|
+
# end
|
74
|
+
def max_workers
|
75
|
+
Resque::Kubernetes.max_workers
|
76
|
+
end
|
77
|
+
|
18
78
|
private
|
19
79
|
|
20
80
|
def jobs_client
|
@@ -22,132 +82,97 @@ module Resque
|
|
22
82
|
@jobs_client = client("/apis/batch")
|
23
83
|
end
|
24
84
|
|
25
|
-
def pods_client
|
26
|
-
return @pods_client if @pods_client
|
27
|
-
@pods_client = client("")
|
28
|
-
end
|
29
|
-
|
30
85
|
def client(scope)
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
{ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"},
|
41
|
-
{bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token"}
|
42
|
-
)
|
43
|
-
elsif File.exist?(kubeconfig)
|
44
|
-
# When running in development, use the config file for `kubectl` and default application credentials
|
45
|
-
kubeconfig = File.join(ENV["HOME"], ".kube", "config")
|
46
|
-
config = Kubeclient::Config.read(kubeconfig)
|
47
|
-
context = Kubeclient::Config::Context.new(
|
48
|
-
config.context.api_endpoint,
|
49
|
-
config.context.api_version,
|
50
|
-
config.context.ssl_options,
|
51
|
-
{use_default_gcp: true}
|
52
|
-
)
|
53
|
-
end
|
54
|
-
|
55
|
-
if context
|
56
|
-
Kubeclient::Client.new(
|
57
|
-
context.api_endpoint + scope,
|
58
|
-
context.api_version,
|
59
|
-
ssl_options: context.ssl_options,
|
60
|
-
auth_options: context.auth_options,
|
61
|
-
)
|
62
|
-
end
|
86
|
+
context = ContextFactory.context
|
87
|
+
return unless context
|
88
|
+
|
89
|
+
Kubeclient::Client.new(
|
90
|
+
context.api_endpoint + scope,
|
91
|
+
context.api_version,
|
92
|
+
ssl_options: context.ssl_options,
|
93
|
+
auth_options: context.auth_options
|
94
|
+
)
|
63
95
|
end
|
64
96
|
|
65
|
-
def
|
97
|
+
def finished_jobs
|
66
98
|
resque_jobs = jobs_client.get_jobs(label_selector: "resque-kubernetes=job")
|
67
|
-
|
68
|
-
|
69
|
-
finished.each do |job|
|
70
|
-
jobs_client.delete_job(job.metadata.name, job.metadata.namespace)
|
71
|
-
end
|
99
|
+
resque_jobs.select { |job| job.spec.completions == job.status.succeeded }
|
72
100
|
end
|
73
101
|
|
74
|
-
def
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
102
|
+
def reap_finished_jobs
|
103
|
+
finished_jobs.each do |job|
|
104
|
+
begin
|
105
|
+
jobs_client.delete_job(job.metadata.name, job.metadata.namespace)
|
106
|
+
rescue KubeException => e
|
107
|
+
raise unless e.error_code == 404
|
108
|
+
end
|
80
109
|
end
|
81
110
|
end
|
82
111
|
|
83
112
|
def apply_kubernetes_job
|
84
|
-
manifest = job_manifest
|
113
|
+
manifest = DeepHash.new.merge!(job_manifest)
|
85
114
|
ensure_namespace(manifest)
|
86
115
|
|
87
116
|
# Do not start job if we have reached our maximum count
|
88
117
|
return if jobs_maxed?(manifest["metadata"]["name"], manifest["metadata"]["namespace"])
|
89
118
|
|
90
|
-
|
91
|
-
ensure_term_on_empty(manifest)
|
92
|
-
ensure_reset_policy(manifest)
|
93
|
-
update_job_name(manifest)
|
119
|
+
adjust_manifest(manifest)
|
94
120
|
|
95
121
|
job = Kubeclient::Resource.new(manifest)
|
96
122
|
jobs_client.create_job(job)
|
97
123
|
end
|
98
124
|
|
99
125
|
def jobs_maxed?(name, namespace)
|
100
|
-
resque_jobs = jobs_client.get_jobs(
|
101
|
-
|
102
|
-
|
126
|
+
resque_jobs = jobs_client.get_jobs(
|
127
|
+
label_selector: "resque-kubernetes=job,resque-kubernetes-group=#{name}",
|
128
|
+
namespace: namespace
|
129
|
+
)
|
130
|
+
running = resque_jobs.reject { |job| job.spec.completions == job.status.succeeded }
|
131
|
+
running.size == max_workers
|
132
|
+
end
|
133
|
+
|
134
|
+
def adjust_manifest(manifest)
|
135
|
+
add_labels(manifest)
|
136
|
+
ensure_term_on_empty(manifest)
|
137
|
+
ensure_reset_policy(manifest)
|
138
|
+
update_job_name(manifest)
|
103
139
|
end
|
104
140
|
|
105
141
|
def add_labels(manifest)
|
106
|
-
manifest[
|
107
|
-
manifest["metadata"]["labels"] ||= {}
|
108
|
-
manifest["metadata"]["labels"]["resque-kubernetes"] = "job"
|
142
|
+
manifest.deep_add(%w[metadata labels resque-kubernetes], "job")
|
109
143
|
manifest["metadata"]["labels"]["resque-kubernetes-group"] = manifest["metadata"]["name"]
|
110
|
-
manifest[
|
111
|
-
manifest["spec"]["template"]["metadata"]["labels"] ||= {}
|
112
|
-
manifest["spec"]["template"]["metadata"]["labels"]["resque-kubernetes"] = "pod"
|
144
|
+
manifest.deep_add(%w[spec template metadata labels resque-kubernetes], "pod")
|
113
145
|
end
|
114
146
|
|
115
147
|
def ensure_term_on_empty(manifest)
|
116
148
|
manifest["spec"]["template"]["spec"] ||= {}
|
117
149
|
manifest["spec"]["template"]["spec"]["containers"] ||= []
|
118
150
|
manifest["spec"]["template"]["spec"]["containers"].each do |container|
|
119
|
-
container
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
151
|
+
container_term_on_empty(container)
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
def container_term_on_empty(container)
|
156
|
+
container["env"] ||= []
|
157
|
+
term_on_empty = container["env"].find { |env| env["name"] == "TERM_ON_EMPTY" }
|
158
|
+
unless term_on_empty
|
159
|
+
term_on_empty = {"name" => "TERM_ON_EMPTY"}
|
160
|
+
container["env"] << term_on_empty
|
126
161
|
end
|
162
|
+
term_on_empty["value"] = "1"
|
127
163
|
end
|
128
164
|
|
129
165
|
def ensure_reset_policy(manifest)
|
130
166
|
manifest["spec"]["template"]["spec"]["restartPolicy"] ||= "OnFailure"
|
131
167
|
end
|
132
168
|
|
133
|
-
|
134
169
|
def ensure_namespace(manifest)
|
135
170
|
manifest["metadata"]["namespace"] ||= "default"
|
136
171
|
end
|
137
172
|
|
138
173
|
def update_job_name(manifest)
|
139
|
-
manifest["metadata"]["name"] += "-#{
|
140
|
-
end
|
141
|
-
|
142
|
-
# Returns an n-length string of characters [a-z0-9]
|
143
|
-
def dns_safe_random(n = 5)
|
144
|
-
s = [SecureRandom.random_bytes(n)].pack("m*")
|
145
|
-
s.delete!("=\n")
|
146
|
-
s.tr!("+/_-", "0")
|
147
|
-
s.tr!("A-Z", "a-z")
|
148
|
-
s[0...n]
|
174
|
+
manifest["metadata"]["name"] += "-#{DNSSafeRandom.random_chars}"
|
149
175
|
end
|
150
|
-
|
151
176
|
end
|
152
177
|
end
|
153
178
|
end
|
@@ -1,7 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "resque"
|
2
4
|
|
3
5
|
module Resque
|
4
6
|
module Kubernetes
|
7
|
+
# Patches the resque worker to terminate when the queue is empty.
|
8
|
+
#
|
9
|
+
# This patch enables setting an environment variable, `TERM_ON_EMPTY`
|
10
|
+
# that causes the worker to shutdown when the queue no longer has any
|
11
|
+
# resque jobs. This allows running workers as Kuberenetes Jobs that will
|
12
|
+
# terminate when there is no longer any work to do.
|
13
|
+
#
|
14
|
+
# To use, make sure that the container images that hold your workers are
|
15
|
+
# built to include the `resque-kubernetes` gem and set `TERM_ON_EMPTY` in
|
16
|
+
# their environment to a truthy value (e.g. "1").
|
5
17
|
module Worker
|
6
18
|
def self.included(base)
|
7
19
|
base.class_eval do
|
@@ -11,6 +23,7 @@ module Resque
|
|
11
23
|
|
12
24
|
attr_accessor :term_on_empty
|
13
25
|
|
26
|
+
# Replace methods on the worker instance
|
14
27
|
module InstanceMethods
|
15
28
|
def prepare
|
16
29
|
self.term_on_empty = ENV["TERM_ON_EMPTY"] if ENV["TERM_ON_EMPTY"]
|
@@ -29,13 +42,11 @@ module Resque
|
|
29
42
|
end
|
30
43
|
end
|
31
44
|
|
32
|
-
|
33
45
|
private
|
34
46
|
|
35
47
|
def queues_empty?
|
36
|
-
queues.all? { |queue| Resque.size(queue)
|
48
|
+
queues.all? { |queue| Resque.size(queue).zero? }
|
37
49
|
end
|
38
|
-
|
39
50
|
end
|
40
51
|
end
|
41
52
|
end
|
data/resque-kubernetes.gemspec
CHANGED
@@ -1,7 +1,8 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path("../lib", __FILE__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
-
require
|
5
|
+
require "resque/kubernetes/version"
|
5
6
|
|
6
7
|
Gem::Specification.new do |spec|
|
7
8
|
spec.name = "resque-kubernetes"
|
@@ -9,8 +10,9 @@ Gem::Specification.new do |spec|
|
|
9
10
|
spec.authors = ["Jeremy Wadsack"]
|
10
11
|
spec.email = ["jeremy.wadsack@gmail.com"]
|
11
12
|
|
12
|
-
spec.summary =
|
13
|
-
spec.description =
|
13
|
+
spec.summary = "Run Resque Jobs as Kubernetes Jobs"
|
14
|
+
spec.description = "Launches a Kubernetes Job when a Resque Job is enqueued, then " \
|
15
|
+
"terminates the worker when there are no more jobs in the queue."
|
14
16
|
spec.homepage = "https://github.com/keylimetoolbox/resque-kubernetes"
|
15
17
|
spec.license = "MIT"
|
16
18
|
|
@@ -21,10 +23,12 @@ Gem::Specification.new do |spec|
|
|
21
23
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
22
24
|
spec.require_paths = ["lib"]
|
23
25
|
|
24
|
-
spec.add_development_dependency "bundler", "~> 1.
|
25
|
-
spec.add_development_dependency "
|
26
|
-
spec.add_development_dependency "
|
26
|
+
spec.add_development_dependency "bundler", "~> 1.16"
|
27
|
+
spec.add_development_dependency "bundler-audit", "~> 0"
|
28
|
+
spec.add_development_dependency "rake", "~> 12.3"
|
29
|
+
spec.add_development_dependency "rspec", "~> 3.7"
|
30
|
+
spec.add_development_dependency "rubocop", "~> 0.52", ">= 0.52.1"
|
27
31
|
|
28
|
-
spec.add_dependency "resque", "~> 1.26"
|
29
32
|
spec.add_dependency "kubeclient", "~> 2.2"
|
33
|
+
spec.add_dependency "resque", "~> 1.26"
|
30
34
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: resque-kubernetes
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jeremy Wadsack
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2018-02-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -16,56 +16,76 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '1.
|
19
|
+
version: '1.16'
|
20
20
|
type: :development
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '1.
|
26
|
+
version: '1.16'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: bundler-audit
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: rake
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
30
44
|
requirements:
|
31
45
|
- - "~>"
|
32
46
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
47
|
+
version: '12.3'
|
34
48
|
type: :development
|
35
49
|
prerelease: false
|
36
50
|
version_requirements: !ruby/object:Gem::Requirement
|
37
51
|
requirements:
|
38
52
|
- - "~>"
|
39
53
|
- !ruby/object:Gem::Version
|
40
|
-
version: '
|
54
|
+
version: '12.3'
|
41
55
|
- !ruby/object:Gem::Dependency
|
42
56
|
name: rspec
|
43
57
|
requirement: !ruby/object:Gem::Requirement
|
44
58
|
requirements:
|
45
59
|
- - "~>"
|
46
60
|
- !ruby/object:Gem::Version
|
47
|
-
version: '3.
|
61
|
+
version: '3.7'
|
48
62
|
type: :development
|
49
63
|
prerelease: false
|
50
64
|
version_requirements: !ruby/object:Gem::Requirement
|
51
65
|
requirements:
|
52
66
|
- - "~>"
|
53
67
|
- !ruby/object:Gem::Version
|
54
|
-
version: '3.
|
68
|
+
version: '3.7'
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
|
-
name:
|
70
|
+
name: rubocop
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
58
72
|
requirements:
|
59
73
|
- - "~>"
|
60
74
|
- !ruby/object:Gem::Version
|
61
|
-
version: '
|
62
|
-
|
75
|
+
version: '0.52'
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: 0.52.1
|
79
|
+
type: :development
|
63
80
|
prerelease: false
|
64
81
|
version_requirements: !ruby/object:Gem::Requirement
|
65
82
|
requirements:
|
66
83
|
- - "~>"
|
67
84
|
- !ruby/object:Gem::Version
|
68
|
-
version: '
|
85
|
+
version: '0.52'
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: 0.52.1
|
69
89
|
- !ruby/object:Gem::Dependency
|
70
90
|
name: kubeclient
|
71
91
|
requirement: !ruby/object:Gem::Requirement
|
@@ -80,6 +100,20 @@ dependencies:
|
|
80
100
|
- - "~>"
|
81
101
|
- !ruby/object:Gem::Version
|
82
102
|
version: '2.2'
|
103
|
+
- !ruby/object:Gem::Dependency
|
104
|
+
name: resque
|
105
|
+
requirement: !ruby/object:Gem::Requirement
|
106
|
+
requirements:
|
107
|
+
- - "~>"
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '1.26'
|
110
|
+
type: :runtime
|
111
|
+
prerelease: false
|
112
|
+
version_requirements: !ruby/object:Gem::Requirement
|
113
|
+
requirements:
|
114
|
+
- - "~>"
|
115
|
+
- !ruby/object:Gem::Version
|
116
|
+
version: '1.26'
|
83
117
|
description: Launches a Kubernetes Job when a Resque Job is enqueued, then terminates
|
84
118
|
the worker when there are no more jobs in the queue.
|
85
119
|
email:
|
@@ -90,9 +124,11 @@ extra_rdoc_files: []
|
|
90
124
|
files:
|
91
125
|
- ".gitignore"
|
92
126
|
- ".rspec"
|
127
|
+
- ".rubocop.yml"
|
93
128
|
- ".ruby-gemset"
|
94
129
|
- ".ruby-version"
|
95
130
|
- ".travis.yml"
|
131
|
+
- CHANGELOG.md
|
96
132
|
- Gemfile
|
97
133
|
- LICENSE.txt
|
98
134
|
- README.md
|
@@ -101,6 +137,9 @@ files:
|
|
101
137
|
- bin/setup
|
102
138
|
- lib/resque/kubernetes.rb
|
103
139
|
- lib/resque/kubernetes/configurable.rb
|
140
|
+
- lib/resque/kubernetes/context_factory.rb
|
141
|
+
- lib/resque/kubernetes/deep_hash.rb
|
142
|
+
- lib/resque/kubernetes/dns_safe_random.rb
|
104
143
|
- lib/resque/kubernetes/job.rb
|
105
144
|
- lib/resque/kubernetes/version.rb
|
106
145
|
- lib/resque/kubernetes/worker.rb
|