kube_queue 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +3 -1
- data/CHANGELOG.md +14 -0
- data/Dockerfile.myapp +3 -0
- data/Guardfile +17 -0
- data/README.md +107 -4
- data/bin/console +2 -2
- data/kube_queue.gemspec +4 -0
- data/lib/active_job/adapters/kube_queue_adapter.rb +6 -4
- data/lib/kube_queue.rb +6 -38
- data/lib/kube_queue/client.rb +12 -6
- data/lib/kube_queue/executor.rb +9 -3
- data/lib/kube_queue/job_specification.rb +24 -3
- data/lib/kube_queue/manifest_builder.rb +22 -5
- data/lib/kube_queue/version.rb +1 -1
- data/lib/kube_queue/worker.rb +122 -10
- data/lib/kube_queue/worker/dsl.rb +32 -0
- data/template/job.yaml +36 -2
- metadata +60 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9ff84943f07b2ea41579cbeeda959625a832a848a84f0b06fd5fc0048cbd590c
|
4
|
+
data.tar.gz: d5cb9f6eba803d1ffc2334c573474665fc025456df3370ee062b0c7f2c7e29ed
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c4fadb77cc08d20596227998e8fdfeb9da9592be117bd2177d7a8674c44fd1b8b003d1dcec5ae11d5c6fcd24ae8611ab1c979fb395f465c22d99c2cfae857431
|
7
|
+
data.tar.gz: 9bff732597d04f6907eaf05557af4aa1e39422311c80efb1d36f219ea71420d0fe873454a868589ba9188f41d465434adc2d1f055624a8cb27d026f11baf4d55
|
data/.rubocop.yml
CHANGED
@@ -6,6 +6,8 @@ AllCops:
|
|
6
6
|
Exclude:
|
7
7
|
- Gemfile
|
8
8
|
- Rakefile
|
9
|
+
- examples/**/*
|
10
|
+
- kube_queue.gemspec
|
9
11
|
DisplayCopNames: true
|
10
12
|
|
11
13
|
Gemspec/OrderedDependencies:
|
@@ -48,7 +50,7 @@ Metrics/MethodLength:
|
|
48
50
|
Max: 15
|
49
51
|
|
50
52
|
Metrics/AbcSize:
|
51
|
-
Max:
|
53
|
+
Max: 25
|
52
54
|
|
53
55
|
Style/ParallelAssignment:
|
54
56
|
Enabled: false
|
data/CHANGELOG.md
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
# CHANGELOG
|
2
|
+
|
3
|
+
## 0.4.0
|
4
|
+
|
5
|
+
- Support scheduled job with Kubernetes CronJob.
|
6
|
+
- Support more settings
|
7
|
+
- `env_from_secret`
|
8
|
+
- `env_from_configmap`
|
9
|
+
- `cpu_limit`, `cpu_request`, `memory_limit`, `memory_request`
|
10
|
+
|
11
|
+
## 0.3.0
|
12
|
+
|
13
|
+
- Support ActiveJob
|
14
|
+
- Change command line interfaces
|
data/Dockerfile.myapp
CHANGED
@@ -17,6 +17,9 @@ COPY examples/myapp/Gemfile examples/myapp/Gemfile.lock $WORKDIR/
|
|
17
17
|
RUN bundle install -j4
|
18
18
|
|
19
19
|
COPY lib $KUBE_QUEUE_PATH/lib
|
20
|
+
COPY template $KUBE_QUEUE_PATH/template
|
20
21
|
COPY examples/myapp/ $WORKDIR
|
21
22
|
|
23
|
+
RUN bundle exec rails assets:precompile
|
24
|
+
|
22
25
|
CMD ["bundle", "exec", "rails", "console"]
|
data/Guardfile
ADDED
@@ -0,0 +1,17 @@
|
|
1
|
+
guard :rspec, cmd: "bundle exec rspec" do
|
2
|
+
require "guard/rspec/dsl"
|
3
|
+
dsl = Guard::RSpec::Dsl.new(self)
|
4
|
+
|
5
|
+
rspec = dsl.rspec
|
6
|
+
watch(rspec.spec_helper) { rspec.spec_dir }
|
7
|
+
watch(rspec.spec_support) { rspec.spec_dir }
|
8
|
+
watch(rspec.spec_files)
|
9
|
+
|
10
|
+
ruby = dsl.ruby
|
11
|
+
dsl.watch_spec_files_for(ruby.lib_files)
|
12
|
+
end
|
13
|
+
|
14
|
+
guard :rubocop do
|
15
|
+
watch(%r{.+\.rb$})
|
16
|
+
watch(%r{(?:.+/)?\.rubocop(?:_todo)?\.yml$}) { |m| File.dirname(m[0]) }
|
17
|
+
end
|
data/README.md
CHANGED
@@ -49,7 +49,10 @@ end
|
|
49
49
|
and run:
|
50
50
|
|
51
51
|
```ruby
|
52
|
-
TestWorker.
|
52
|
+
TestWorker.enqueue(message: 'hello')
|
53
|
+
|
54
|
+
# delay
|
55
|
+
TestWorker.enqueue_at(message: 'hello', Time.now + 100)
|
53
56
|
```
|
54
57
|
|
55
58
|
### ActiveJob Support
|
@@ -89,7 +92,7 @@ irb(main):003:0> job.status
|
|
89
92
|
=> #<K8s::Resource conditions=[{:type=>"Complete", :status=>"True", :lastProbeTime=>"2019-08-12T15:57:03Z", :lastTransitionTime=>"2019-08-12T15:57:03Z"}], startTime="2019-08-12T15:56:37Z", completionTime="2019-08-12T15:57:03Z", succeeded=1>
|
90
93
|
```
|
91
94
|
|
92
|
-
See more examples in [here](examples).
|
95
|
+
See more examples in [here](examples/myapp/app/jobs).
|
93
96
|
|
94
97
|
### Run job on locally
|
95
98
|
|
@@ -97,13 +100,113 @@ See more examples in [here](examples).
|
|
97
100
|
bundle exec kube_queue runner JOB_NAME [PAYLOAD]
|
98
101
|
```
|
99
102
|
|
100
|
-
See more information by `kube_queue help` or [here](exe/kube_queue).
|
103
|
+
See more information by `kube_queue help` or read [here](exe/kube_queue).
|
104
|
+
|
105
|
+
## Advanced Tips
|
106
|
+
|
107
|
+
### Get a job status
|
108
|
+
|
109
|
+
```ruby
|
110
|
+
job = ComputePiJob.perform_later
|
111
|
+
job.status
|
112
|
+
```
|
113
|
+
|
114
|
+
scheduled job dosent supported now.
|
115
|
+
|
116
|
+
### Check a generating manifest
|
117
|
+
|
118
|
+
```ruby
|
119
|
+
# from class
|
120
|
+
puts ComputePiJob.manifest
|
121
|
+
|
122
|
+
# from instance
|
123
|
+
job = ComputePiJob.perform_later
|
124
|
+
puts job.manifest
|
125
|
+
```
|
126
|
+
|
127
|
+
### Retry job
|
128
|
+
|
129
|
+
Kubernetes Job has a own retry mechanism, if set backoff_limit and/or restart_policy to use it.
|
130
|
+
|
131
|
+
```ruby
|
132
|
+
class ComputePiJob
|
133
|
+
include KubeQueue::Worker
|
134
|
+
|
135
|
+
worker_name 'pi'
|
136
|
+
image 'perl'
|
137
|
+
container_name 'pi'
|
138
|
+
command "perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"
|
139
|
+
|
140
|
+
backoff_limit 10
|
141
|
+
restart_policy 'Never'
|
142
|
+
end
|
143
|
+
```
|
144
|
+
|
145
|
+
More information, see the official document [here](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#pod-backoff-failure-policy).
|
146
|
+
|
147
|
+
### Timeout
|
148
|
+
|
149
|
+
Kubernetes Job has a own timeout mechanism, if set the active_deadline_seconds to use it.
|
150
|
+
|
151
|
+
```ruby
|
152
|
+
class ComputePiJob
|
153
|
+
include KubeQueue::Worker
|
154
|
+
|
155
|
+
worker_name 'pi'
|
156
|
+
image 'perl'
|
157
|
+
container_name 'pi'
|
158
|
+
command "perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"
|
159
|
+
|
160
|
+
active_deadline_seconds 300
|
161
|
+
end
|
162
|
+
```
|
163
|
+
|
164
|
+
More information, see the official document [here](https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#job-termination-and-cleanup).
|
165
|
+
|
166
|
+
### Managing container resources
|
167
|
+
|
168
|
+
When you specify a Pod, you can optional specify hou much CPU and memory container needs.
|
169
|
+
|
170
|
+
```ruby
|
171
|
+
class ComputePiJob
|
172
|
+
include KubeQueue::Worker
|
173
|
+
|
174
|
+
worker_name 'pi'
|
175
|
+
image 'perl'
|
176
|
+
container_name 'pi'
|
177
|
+
command "perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"
|
178
|
+
|
179
|
+
cpu_limit '0.3'
|
180
|
+
cpu_request '0.2'
|
181
|
+
memory_limit '100m'
|
182
|
+
memory_request '50m'
|
183
|
+
end
|
184
|
+
```
|
185
|
+
|
186
|
+
More information, see the official document [here](https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/).
|
187
|
+
|
188
|
+
### Use environment variable from ConfigMap/Secret
|
189
|
+
|
190
|
+
```ruby
|
191
|
+
class ComputePiJob
|
192
|
+
include KubeQueue::Worker
|
193
|
+
|
194
|
+
worker_name 'pi'
|
195
|
+
image 'perl'
|
196
|
+
container_name 'pi'
|
197
|
+
command "perl", "-Mbignum=bpi", "-wle", "print bpi(2000)"
|
198
|
+
|
199
|
+
env_from_secret 'mysecret1', 'mysecret2'
|
200
|
+
env_from_config_map 'myapp'
|
201
|
+
end
|
202
|
+
```
|
101
203
|
|
102
204
|
## Features
|
103
205
|
|
104
206
|
- Add tests.
|
105
207
|
- Support multiple kubernetes client configuration.
|
106
208
|
- Logging informations.
|
209
|
+
- Support to get CronJob status.
|
107
210
|
|
108
211
|
## Development(on GCP/GKE)
|
109
212
|
|
@@ -126,5 +229,5 @@ run:
|
|
126
229
|
```
|
127
230
|
K8S_URL=https://xx.xxx.xxx.xxx K8S_CA_CERT_FILE=$(pwd)/secrets/ca.crt K8S_TOKEN=$(pwd)/secrets/token IMAGE_NAME=gcr.io/your-project/kube-queue bin/console
|
128
231
|
|
129
|
-
irb(main):001:0> TestWorker.
|
232
|
+
irb(main):001:0> TestWorker.enqueue(message: 'hello, kubernetes!')
|
130
233
|
```
|
data/bin/console
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
require "bundler/setup"
|
4
4
|
require "kube_queue"
|
5
|
-
require_relative "../docker/test_worker"
|
5
|
+
require_relative "../examples/docker/test_worker"
|
6
6
|
|
7
7
|
# You can add fixtures and/or initialization code here to make experimenting
|
8
8
|
# with your gem easier. You can also use a different console, if you like.
|
@@ -16,7 +16,7 @@ require "irb"
|
|
16
16
|
KubeQueue.kubernetes_configure do |client|
|
17
17
|
client.url = ENV['K8S_URL']
|
18
18
|
client.ssl_ca_file = ENV['K8S_CA_CERT_FILE']
|
19
|
-
client.auth_token = ENV['
|
19
|
+
client.auth_token = File.read(ENV['K8S_BEARER_TOKEN_FILE'])
|
20
20
|
end
|
21
21
|
|
22
22
|
IRB.start(__FILE__)
|
data/kube_queue.gemspec
CHANGED
@@ -34,4 +34,8 @@ Gem::Specification.new do |spec|
|
|
34
34
|
spec.add_development_dependency "rubocop"
|
35
35
|
spec.add_development_dependency "rubocop-performance"
|
36
36
|
spec.add_development_dependency "pry-byebug"
|
37
|
+
spec.add_development_dependency "guard"
|
38
|
+
spec.add_development_dependency "guard-rspec"
|
39
|
+
spec.add_development_dependency "guard-rubocop"
|
40
|
+
spec.add_development_dependency "activejob"
|
37
41
|
end
|
@@ -10,11 +10,12 @@ module ActiveJob
|
|
10
10
|
class << self
|
11
11
|
# Interface for ActiveJob 4.2
|
12
12
|
def enqueue(job)
|
13
|
-
KubeQueue.executor.enqueue(job
|
13
|
+
KubeQueue.executor.enqueue(job)
|
14
14
|
end
|
15
15
|
|
16
|
-
def enqueue_at(
|
17
|
-
|
16
|
+
def enqueue_at(job, timestamp)
|
17
|
+
job.scheduled_at = timestamp
|
18
|
+
KubeQueue.executor.enqueue(job)
|
18
19
|
end
|
19
20
|
end
|
20
21
|
|
@@ -24,7 +25,8 @@ module ActiveJob
|
|
24
25
|
end
|
25
26
|
|
26
27
|
def enqueue_at(job, timestamp)
|
27
|
-
|
28
|
+
job.scheduled_at = timestamp
|
29
|
+
KubeQueueAdapter.enqueue(job)
|
28
30
|
end
|
29
31
|
end
|
30
32
|
end
|
data/lib/kube_queue.rb
CHANGED
@@ -9,7 +9,7 @@ module KubeQueue
|
|
9
9
|
class JobNotFound < StandardError; end
|
10
10
|
|
11
11
|
class << self
|
12
|
-
attr_writer :executor
|
12
|
+
attr_writer :executor, :client
|
13
13
|
|
14
14
|
def executor
|
15
15
|
@executor ||= default_executor
|
@@ -20,13 +20,17 @@ module KubeQueue
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def client
|
23
|
-
@client ||=
|
23
|
+
@client ||= default_client
|
24
24
|
end
|
25
25
|
|
26
26
|
def default_executor
|
27
27
|
Executor.new
|
28
28
|
end
|
29
29
|
|
30
|
+
def default_client
|
31
|
+
Client.new
|
32
|
+
end
|
33
|
+
|
30
34
|
def configure(&block)
|
31
35
|
configuration.configure(&block)
|
32
36
|
end
|
@@ -35,18 +39,6 @@ module KubeQueue
|
|
35
39
|
@configuration ||= Configuration.new
|
36
40
|
end
|
37
41
|
|
38
|
-
def list(namespace = nil)
|
39
|
-
client.list_job(namespace).map do |job|
|
40
|
-
worker = fetch_worker(job.metadata.annotations['kube-queue-job-class'])
|
41
|
-
job_id = job.metadata.annotations['kube-queue-job-id']
|
42
|
-
payload = deserialize(job.metadata.annotations['kube-queue-message-payload'])
|
43
|
-
|
44
|
-
job = worker.new(*payload)
|
45
|
-
job.job_id = job_id
|
46
|
-
job
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
42
|
attr_writer :default_env
|
51
43
|
|
52
44
|
def default_env
|
@@ -71,29 +63,5 @@ module KubeQueue
|
|
71
63
|
def worker_registry
|
72
64
|
@worker_registry ||= {}
|
73
65
|
end
|
74
|
-
|
75
|
-
private
|
76
|
-
|
77
|
-
def deserialize(payload)
|
78
|
-
return payload if payload.blank?
|
79
|
-
|
80
|
-
payload = JSON.parse(payload)
|
81
|
-
# Compatibility for ActiveJob serialized payload
|
82
|
-
payload = [payload] unless payload.is_a?(Array)
|
83
|
-
|
84
|
-
if defined?(ActiveJob::Arguments)
|
85
|
-
begin
|
86
|
-
payload = ActiveJob::Arguments.deserialize(payload)
|
87
|
-
rescue ActiveJob::DeserializationError => e
|
88
|
-
logger.warn e.message
|
89
|
-
logger.warn "#{payload} can not deserialized"
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
|
-
payload
|
94
|
-
rescue JSON::ParseError => e
|
95
|
-
logger.warn e.message
|
96
|
-
logger.warn "#{payload} can not deserialized"
|
97
|
-
end
|
98
66
|
end
|
99
67
|
end
|
data/lib/kube_queue/client.rb
CHANGED
@@ -6,16 +6,22 @@ module KubeQueue
|
|
6
6
|
def create_job(manifest)
|
7
7
|
job = K8s::Resource.new(manifest)
|
8
8
|
job.metadata.namespace ||= 'default'
|
9
|
-
client.resource('jobs').create_resource(job)
|
9
|
+
client.api('batch/v1').resource('jobs').create_resource(job)
|
10
10
|
end
|
11
11
|
|
12
12
|
def get_job(namespace, name)
|
13
|
-
client.resource('jobs', namespace: namespace).get(name)
|
13
|
+
client.api('batch/v1').resource('jobs', namespace: namespace).get(name)
|
14
14
|
end
|
15
15
|
|
16
|
-
def list_job(namespace = nil)
|
17
|
-
selector = { 'kube-queue-job': 'true' }
|
18
|
-
client.resource('jobs', namespace: namespace).list(labelSelector: selector)
|
16
|
+
def list_job(job_class, namespace = nil)
|
17
|
+
selector = { 'kube-queue-job': 'true', 'kube-queue-job-class': job_class }
|
18
|
+
client.api('batch/v1').resource('jobs', namespace: namespace).list(labelSelector: selector)
|
19
|
+
end
|
20
|
+
|
21
|
+
def create_cron_job(manifest)
|
22
|
+
cron_job = K8s::Resource.new(manifest)
|
23
|
+
cron_job.metadata.namespace ||= 'default'
|
24
|
+
client.api('batch/v1beta1').resource('cronjobs').create_resource(cron_job)
|
19
25
|
end
|
20
26
|
|
21
27
|
attr_accessor :url, :ssl_ca_file, :auth_token
|
@@ -23,7 +29,7 @@ module KubeQueue
|
|
23
29
|
private
|
24
30
|
|
25
31
|
def client
|
26
|
-
@client ||= K8s.client(url, ssl_ca_file: ssl_ca_file, auth_token: auth_token)
|
32
|
+
@client ||= K8s.client(url, ssl_ca_file: ssl_ca_file, auth_token: auth_token)
|
27
33
|
end
|
28
34
|
end
|
29
35
|
end
|
data/lib/kube_queue/executor.rb
CHANGED
@@ -2,9 +2,15 @@ require 'kube_queue/manifest_builder'
|
|
2
2
|
|
3
3
|
module KubeQueue
|
4
4
|
class Executor
|
5
|
-
def enqueue(job
|
6
|
-
|
7
|
-
|
5
|
+
def enqueue(job)
|
6
|
+
resource = if job.scheduled_at
|
7
|
+
KubeQueue.client.create_cron_job(job.manifest)
|
8
|
+
else
|
9
|
+
KubeQueue.client.create_job(job.manifest)
|
10
|
+
end
|
11
|
+
|
12
|
+
job.resource = resource
|
13
|
+
resource
|
8
14
|
end
|
9
15
|
end
|
10
16
|
end
|
@@ -5,13 +5,14 @@ module KubeQueue
|
|
5
5
|
class JobSpecification
|
6
6
|
class MissingParameterError < StandardError; end
|
7
7
|
|
8
|
-
|
9
8
|
attr_reader :job_class
|
10
9
|
|
11
|
-
attr_accessor :payload, :name, :active_deadline_seconds, :backoff_limit
|
10
|
+
attr_accessor :payload, :name, :active_deadline_seconds, :backoff_limit, :starting_deadline_seconds,
|
11
|
+
:cpu_limit, :memory_limit, :cpu_request, :memory_request
|
12
12
|
|
13
13
|
attr_writer :image, :namespace, :worker_name, :command,
|
14
|
-
:container_name, :restart_policy, :job_labels, :pod_labels
|
14
|
+
:container_name, :restart_policy, :job_labels, :pod_labels,
|
15
|
+
:env_from_config_map, :env_from_secret, :concurrent_policy
|
15
16
|
|
16
17
|
def initialize(job_class)
|
17
18
|
@job_class = job_class
|
@@ -57,6 +58,26 @@ module KubeQueue
|
|
57
58
|
KubeQueue.default_env.merge(@env || {})
|
58
59
|
end
|
59
60
|
|
61
|
+
def env_from_config_map
|
62
|
+
@env_from_config_map || []
|
63
|
+
end
|
64
|
+
|
65
|
+
def env_from_secret
|
66
|
+
@env_from_config_map || []
|
67
|
+
end
|
68
|
+
|
69
|
+
def env_from_exists?
|
70
|
+
!env_from_config_map.empty? && !env_from_secret.empty?
|
71
|
+
end
|
72
|
+
|
73
|
+
def concurrent_policy
|
74
|
+
@concurrent_policy || 'Allow'
|
75
|
+
end
|
76
|
+
|
77
|
+
def resources_exists?
|
78
|
+
@cpu_limit || @memory_limit || @cpu_request || @memory_request
|
79
|
+
end
|
80
|
+
|
60
81
|
def raise_not_found_required_parameter(field)
|
61
82
|
raise MissingParameterError, "#{field} is required"
|
62
83
|
end
|
@@ -5,9 +5,8 @@ module KubeQueue
|
|
5
5
|
class ManifestBuilder
|
6
6
|
attr_reader :job
|
7
7
|
|
8
|
-
def initialize(job
|
8
|
+
def initialize(job)
|
9
9
|
@job = job
|
10
|
-
@payload = payload
|
11
10
|
end
|
12
11
|
|
13
12
|
def spec
|
@@ -15,11 +14,29 @@ module KubeQueue
|
|
15
14
|
end
|
16
15
|
|
17
16
|
def payload
|
18
|
-
|
17
|
+
JSON.generate(job.serialized_payload, quirks_mode: true)
|
19
18
|
end
|
20
19
|
|
21
|
-
def
|
22
|
-
YAML.safe_load(ERB.new(job.
|
20
|
+
def build_job
|
21
|
+
YAML.safe_load(ERB.new(job.read_template, nil, "-").result(binding))
|
22
|
+
end
|
23
|
+
|
24
|
+
def build_cron_job(cron)
|
25
|
+
template = YAML.safe_load(ERB.new(job.read_template, nil, "-").result(binding))
|
26
|
+
|
27
|
+
{
|
28
|
+
apiVersion: "batch/v1beta1",
|
29
|
+
kind: "CronJob",
|
30
|
+
metadata: template["metadata"],
|
31
|
+
spec: {
|
32
|
+
startingDeadlineSeconds: job.job_spec.starting_deadline_seconds,
|
33
|
+
concurrentPolicy: job.job_spec.concurrent_policy,
|
34
|
+
schedule: cron,
|
35
|
+
jobTemplate: {
|
36
|
+
spec: template["spec"]
|
37
|
+
}
|
38
|
+
}
|
39
|
+
}
|
23
40
|
end
|
24
41
|
end
|
25
42
|
end
|
data/lib/kube_queue/version.rb
CHANGED
data/lib/kube_queue/worker.rb
CHANGED
@@ -12,21 +12,82 @@ module KubeQueue
|
|
12
12
|
module ClassMethods
|
13
13
|
include DSL
|
14
14
|
|
15
|
-
def
|
16
|
-
|
15
|
+
def list
|
16
|
+
namespace = job_spec.namespace
|
17
|
+
|
18
|
+
KubeQueue.client.list_job(job_spec.job_class, namespace).map do |res|
|
19
|
+
worker = KubeQueue.fetch_worker(res.metadata.annotations['kube-queue-job-class'])
|
20
|
+
job = worker.new(*payload)
|
21
|
+
job.resource = res
|
22
|
+
job
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def find(job_id)
|
27
|
+
namespace = job_spec.namespace
|
28
|
+
|
29
|
+
name = job_spec.job_name(job_id)
|
30
|
+
|
31
|
+
res = KubeQueue.client.get_job(name, namespace)
|
32
|
+
worker = KubeQueue.fetch_worker(res.metadata.annotations['kube-queue-job-class'])
|
33
|
+
|
34
|
+
payload = deserialize_annotation_payload(res.annotations['kube-queue-job-payload'])
|
35
|
+
|
36
|
+
job = worker.new(*payload)
|
37
|
+
job.resource = res
|
38
|
+
job
|
39
|
+
end
|
40
|
+
|
41
|
+
def enqueue(*args)
|
42
|
+
job = new(*args)
|
43
|
+
KubeQueue.executor.enqueue(job)
|
44
|
+
job
|
17
45
|
end
|
18
46
|
alias_method :perform_async, :enqueue
|
19
47
|
|
20
|
-
def enqueue_at(
|
21
|
-
|
48
|
+
def enqueue_at(*args)
|
49
|
+
args = args.dup
|
50
|
+
timestamp = args.pop
|
51
|
+
job = new(*args)
|
52
|
+
job.scheduled_at = timestamp
|
53
|
+
KubeQueue.executor.enqueue(job)
|
54
|
+
job
|
22
55
|
end
|
23
56
|
|
24
57
|
def read_template
|
25
58
|
File.read(@template || File.expand_path('../../../template/job.yaml', __FILE__))
|
26
59
|
end
|
60
|
+
|
61
|
+
def manifest
|
62
|
+
new.manifest
|
63
|
+
end
|
64
|
+
|
65
|
+
private
|
66
|
+
|
67
|
+
def deserialize_annotation_payload(payload)
|
68
|
+
return payload if payload.empty?
|
69
|
+
|
70
|
+
payload = JSON.parse(payload)
|
71
|
+
# Compatibility for ActiveJob serialized payload
|
72
|
+
payload = [payload] unless payload.is_a?(Array)
|
73
|
+
|
74
|
+
if defined?(ActiveJob::Arguments)
|
75
|
+
begin
|
76
|
+
payload = ActiveJob::Arguments.deserialize(payload)
|
77
|
+
rescue ActiveJob::DeserializationError => e
|
78
|
+
logger.error e.message
|
79
|
+
logger.error "#{payload} can not deserialized"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
payload
|
84
|
+
rescue JSON::ParseError => e
|
85
|
+
logger.error e.message
|
86
|
+
logger.error "#{payload} can not deserialized"
|
87
|
+
end
|
27
88
|
end
|
28
89
|
|
29
|
-
def
|
90
|
+
def read_template
|
30
91
|
self.class.read_template
|
31
92
|
end
|
32
93
|
|
@@ -34,14 +95,22 @@ module KubeQueue
|
|
34
95
|
self.class.job_spec
|
35
96
|
end
|
36
97
|
|
37
|
-
attr_accessor :job_id, :
|
98
|
+
attr_accessor :job_id, :scheduled_at
|
99
|
+
attr_reader :arguments, :resource
|
100
|
+
|
101
|
+
alias_method :payload, :arguments
|
38
102
|
|
39
103
|
def initialize(*arguments)
|
40
104
|
# Compatibility for ActiveJob interface
|
41
|
-
|
105
|
+
if method(__method__).super_method.arity.zero?
|
106
|
+
super()
|
107
|
+
else
|
108
|
+
super
|
109
|
+
end
|
42
110
|
|
43
111
|
@arguments = arguments
|
44
112
|
@job_id = SecureRandom.uuid
|
113
|
+
@loaded = false
|
45
114
|
end
|
46
115
|
|
47
116
|
def perform_now
|
@@ -55,10 +124,53 @@ module KubeQueue
|
|
55
124
|
raise NotImplementedError
|
56
125
|
end
|
57
126
|
|
58
|
-
# FIXME: improve performance
|
59
127
|
def status
|
60
|
-
|
61
|
-
|
128
|
+
return @resource.status if loaded?
|
129
|
+
|
130
|
+
load_target
|
131
|
+
|
132
|
+
@resource.status
|
133
|
+
end
|
134
|
+
|
135
|
+
def loaded?
|
136
|
+
@loaded
|
137
|
+
end
|
138
|
+
|
139
|
+
def reload!
|
140
|
+
@loaded = false
|
141
|
+
@resource = nil
|
142
|
+
|
143
|
+
load_target
|
144
|
+
end
|
145
|
+
|
146
|
+
def manifest
|
147
|
+
if scheduled_at
|
148
|
+
# Kubernetes CronJob does not support timezone
|
149
|
+
cron = Time.at(scheduled_at).utc.strftime("%M %H %d %m %w")
|
150
|
+
ManifestBuilder.new(self).build_cron_job(cron)
|
151
|
+
else
|
152
|
+
ManifestBuilder.new(self).build_job
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def serialized_payload
|
157
|
+
if defined?(ActiveJob::Arguments)
|
158
|
+
ActiveJob::Arguments.serialize(arguments)
|
159
|
+
else
|
160
|
+
arguments
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
def resource=(resource)
|
165
|
+
@resource = resource
|
166
|
+
self.job_id = resource.metadata.annotations['kube-queue-job-id']
|
167
|
+
end
|
168
|
+
|
169
|
+
private
|
170
|
+
|
171
|
+
def load_target
|
172
|
+
self.resource = KubeQueue.client.get_job(job_spec.namespace, job_spec.job_name(job_id))
|
173
|
+
@loaded = true
|
62
174
|
end
|
63
175
|
end
|
64
176
|
end
|
@@ -44,6 +44,38 @@ module KubeQueue
|
|
44
44
|
def labels(labels)
|
45
45
|
job_spec.labels = labels
|
46
46
|
end
|
47
|
+
|
48
|
+
def env_from_config_map(*config_map_names)
|
49
|
+
job_spec.env_from_config_map = config_map_names
|
50
|
+
end
|
51
|
+
|
52
|
+
def env_from_secret(*secret_names)
|
53
|
+
job_spec.env_from_config_map = secret_names
|
54
|
+
end
|
55
|
+
|
56
|
+
def cpu_limit(limit)
|
57
|
+
job_spec.cpu_limit = limit
|
58
|
+
end
|
59
|
+
|
60
|
+
def memory_limit(limit)
|
61
|
+
job_spec.memory_limit = limit
|
62
|
+
end
|
63
|
+
|
64
|
+
def cpu_request(request)
|
65
|
+
job_spec.cpu_request = request
|
66
|
+
end
|
67
|
+
|
68
|
+
def memory_request(request)
|
69
|
+
job_spec.memory_request = request
|
70
|
+
end
|
71
|
+
|
72
|
+
def starting_deadline_seconds(seconds)
|
73
|
+
job_spec.starting_deadline_seconds = seconds
|
74
|
+
end
|
75
|
+
|
76
|
+
def concurrent_policy(policy)
|
77
|
+
job_spec.concurrent_policy = policy
|
78
|
+
end
|
47
79
|
end
|
48
80
|
end
|
49
81
|
end
|
data/template/job.yaml
CHANGED
@@ -2,9 +2,9 @@ apiVersion: batch/v1
|
|
2
2
|
kind: Job
|
3
3
|
metadata:
|
4
4
|
annotations:
|
5
|
-
kube-queue-message-payload: '<%= payload %>'
|
6
5
|
kube-queue-job-class: "<%= spec.job_class %>"
|
7
6
|
kube-queue-job-id: "<%= job.job_id %>"
|
7
|
+
kube-queue-job-payload: '<%= payload %>'
|
8
8
|
name: "<%= spec.job_name(job.job_id) %>"
|
9
9
|
namespace: <%= spec.namespace %>
|
10
10
|
labels:
|
@@ -16,9 +16,9 @@ spec:
|
|
16
16
|
template:
|
17
17
|
metadata:
|
18
18
|
annotations:
|
19
|
-
kube-queue-message-payload: '<%= payload %>'
|
20
19
|
kube-queue-job-class: "<%= spec.job_class %>"
|
21
20
|
kube-queue-job-id: "<%= job.job_id %>"
|
21
|
+
kube-queue-job-payload: '<%= payload %>'
|
22
22
|
labels:
|
23
23
|
kube-queue-job: "true"
|
24
24
|
kube-queue-worker-name: "<%= spec.worker_name %>"
|
@@ -38,6 +38,40 @@ spec:
|
|
38
38
|
<%- spec.env.each do |key, value| %>
|
39
39
|
- name: "<%= key %>"
|
40
40
|
value: "<%= value %>"
|
41
|
+
<%- end %>
|
42
|
+
<%- if spec.env_from_exists? %>
|
43
|
+
envFrom:
|
44
|
+
<%- spec.env_from_config_map.each do |name| %>
|
45
|
+
- configMapRef:
|
46
|
+
name: "<%= name %>"
|
47
|
+
<%- end %>
|
48
|
+
<%- spec.env_from_secret.each do |name| %>
|
49
|
+
- secretRef:
|
50
|
+
name: "<%= name %>"
|
51
|
+
<%- end %>
|
52
|
+
<%- end %>
|
53
|
+
<%- if spec.resources_exists? %>
|
54
|
+
resources:
|
55
|
+
<%- if spec.cpu_limit || spec.memory_limit %>
|
56
|
+
limits:
|
57
|
+
<%- if spec.cpu_limit %>
|
58
|
+
cpu: <%= spec.cpu_limit %>
|
59
|
+
<%- end %>
|
60
|
+
<%- if spec.memory_limit %>
|
61
|
+
memory: <%= spec.memory_limit %>
|
62
|
+
<%- end %>
|
63
|
+
<%- end %>
|
64
|
+
<%- if spec.cpu_request || spec.memory_request %>
|
65
|
+
requests:
|
66
|
+
<%- if spec.cpu_request %>
|
67
|
+
cpu: <%= spec.cpu_request %>
|
68
|
+
<%- end %>
|
69
|
+
<%- if spec.memory_request %>
|
70
|
+
memory: <%= spec.memory_request %>
|
71
|
+
<%- end %>
|
72
|
+
<%- end %>
|
73
|
+
<%- else %>
|
74
|
+
resources: {}
|
41
75
|
<%- end %>
|
42
76
|
restartPolicy: "<%= spec.restart_policy %>"
|
43
77
|
backoffLimit: <%= spec.backoff_limit %>
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: kube_queue
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- yuemori
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-08-
|
11
|
+
date: 2019-08-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: k8s-client
|
@@ -150,6 +150,62 @@ dependencies:
|
|
150
150
|
- - ">="
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: guard
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ">="
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '0'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: guard-rspec
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - ">="
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
174
|
+
type: :development
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - ">="
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '0'
|
181
|
+
- !ruby/object:Gem::Dependency
|
182
|
+
name: guard-rubocop
|
183
|
+
requirement: !ruby/object:Gem::Requirement
|
184
|
+
requirements:
|
185
|
+
- - ">="
|
186
|
+
- !ruby/object:Gem::Version
|
187
|
+
version: '0'
|
188
|
+
type: :development
|
189
|
+
prerelease: false
|
190
|
+
version_requirements: !ruby/object:Gem::Requirement
|
191
|
+
requirements:
|
192
|
+
- - ">="
|
193
|
+
- !ruby/object:Gem::Version
|
194
|
+
version: '0'
|
195
|
+
- !ruby/object:Gem::Dependency
|
196
|
+
name: activejob
|
197
|
+
requirement: !ruby/object:Gem::Requirement
|
198
|
+
requirements:
|
199
|
+
- - ">="
|
200
|
+
- !ruby/object:Gem::Version
|
201
|
+
version: '0'
|
202
|
+
type: :development
|
203
|
+
prerelease: false
|
204
|
+
version_requirements: !ruby/object:Gem::Requirement
|
205
|
+
requirements:
|
206
|
+
- - ">="
|
207
|
+
- !ruby/object:Gem::Version
|
208
|
+
version: '0'
|
153
209
|
description: A background job processing with Kubernetes job for Ruby
|
154
210
|
email:
|
155
211
|
- yuemori@aiming-inc.com
|
@@ -164,10 +220,12 @@ files:
|
|
164
220
|
- ".rspec"
|
165
221
|
- ".rubocop.yml"
|
166
222
|
- ".travis.yml"
|
223
|
+
- CHANGELOG.md
|
167
224
|
- CODE_OF_CONDUCT.md
|
168
225
|
- Dockerfile
|
169
226
|
- Dockerfile.myapp
|
170
227
|
- Gemfile
|
228
|
+
- Guardfile
|
171
229
|
- LICENSE.txt
|
172
230
|
- README.md
|
173
231
|
- Rakefile
|