active_job_k8s 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -1
- data/README.md +33 -23
- data/active_job_k8s.gemspec +2 -2
- data/bin/setup +68 -48
- data/lib/active_job/queue_adapters/k8s_adapter.rb +2 -4
- data/lib/active_job_k8s/railtie.rb +12 -0
- data/lib/active_job_k8s/scheduler.rb +41 -6
- data/lib/active_job_k8s/version.rb +1 -1
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 57feaec27f10c9503c7e9455b30526d080b446eb9194f93a7a20e8830b3f3ceb
|
4
|
+
data.tar.gz: a1447113bfcdae14dae4e870e859df4997ae1ab8d6a01dc3dd11351301a7a165
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: feb62f548aaeffade90f53014060463959c3d672d674f348a82474271291b2f7d00e3bb68a6398a26ff7a77de04b7d137df633ad42141bd1623fbc2b785024bf
|
7
|
+
data.tar.gz: 203f78073d6ce7667ab48facf91262b6cab587142ce8afe1b557220e64dcd574ff093ee160e38e6f1eccbe1b4d23177718b524b0bd13eb548ce75282f35f133b
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -4,7 +4,7 @@ WIP gem to make active job work with kubernetes jobs.
|
|
4
4
|
|
5
5
|
Roadmap for V1.0:
|
6
6
|
- [x] ActiveJob.perform_later create a Job in k8s that will execute the job
|
7
|
-
- [
|
7
|
+
- [x] ActiveJob.perform_later with delay create a Job in k8s in suspended mode,
|
8
8
|
a task will enable it as soon as the time is reached
|
9
9
|
- [ ] Limiting the number of concurrent jobs (if there are more jobs they will be created in suspended mode)
|
10
10
|
|
@@ -66,8 +66,33 @@ Rails.application.configure do
|
|
66
66
|
namespace
|
67
67
|
)
|
68
68
|
|
69
|
+
default_manifest = YAML.safe_load(
|
70
|
+
<<~MANIFEST
|
71
|
+
apiVersion: batch/v1
|
72
|
+
kind: Job
|
73
|
+
metadata:
|
74
|
+
name: scheduled-job-name
|
75
|
+
namespace: default
|
76
|
+
spec:
|
77
|
+
template:
|
78
|
+
spec:
|
79
|
+
restartPolicy: Never
|
80
|
+
containers:
|
81
|
+
- name: app-job
|
82
|
+
image: image_of_the_rails_application
|
83
|
+
imagePullPolicy: IfNotPresent
|
84
|
+
command:
|
85
|
+
- /bin/bash
|
86
|
+
args:
|
87
|
+
- '-c'
|
88
|
+
- bundle exec rails active_job_k8s:run_job
|
89
|
+
|
90
|
+
MANIFEST
|
91
|
+
)
|
92
|
+
|
69
93
|
config.active_job.queue_adapter = ActiveJob::QueueAdapters::K8sAdapter.new(
|
70
|
-
kubeclient_context: context
|
94
|
+
kubeclient_context: context,
|
95
|
+
default_manifest: default_manifest
|
71
96
|
)
|
72
97
|
|
73
98
|
end
|
@@ -123,27 +148,12 @@ class HelloWorldJob < ApplicationJob
|
|
123
148
|
def perform(*args)
|
124
149
|
Rails.logger.debug "Hello World"
|
125
150
|
end
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
metadata:
|
133
|
-
name: scheduled-job-name
|
134
|
-
namespace: default
|
135
|
-
spec:
|
136
|
-
template:
|
137
|
-
spec:
|
138
|
-
restartPolicy: Never
|
139
|
-
containers:
|
140
|
-
- name: app-job
|
141
|
-
image: image_of_the_rails_application
|
142
|
-
imagePullPolicy: IfNotPresent
|
143
|
-
|
144
|
-
MANIFEST
|
145
|
-
)
|
146
|
-
end
|
151
|
+
|
152
|
+
#[Optional] if not present it will be taken from scheduler configuration
|
153
|
+
# def manifest
|
154
|
+
# .....
|
155
|
+
# end
|
156
|
+
|
147
157
|
end
|
148
158
|
```
|
149
159
|
|
data/active_job_k8s.gemspec
CHANGED
@@ -9,13 +9,13 @@ Gem::Specification.new do |spec|
|
|
9
9
|
spec.summary = "ActiveJob adapter for kubernetes job"
|
10
10
|
spec.homepage = "https://github.com/oniram88/active_job_k8s"
|
11
11
|
spec.license = "MIT"
|
12
|
-
spec.required_ruby_version = Gem::Requirement.new(">= 2.
|
12
|
+
spec.required_ruby_version = Gem::Requirement.new(">= 2.6.0")
|
13
13
|
|
14
14
|
spec.metadata["allowed_push_host"] = "https://rubygems.org"
|
15
15
|
|
16
16
|
spec.metadata["homepage_uri"] = spec.homepage
|
17
17
|
spec.metadata["source_code_uri"] = spec.homepage
|
18
|
-
spec.metadata["changelog_uri"] = "#{spec.homepage}/CHANGELOG.md"
|
18
|
+
spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/master/CHANGELOG.md"
|
19
19
|
|
20
20
|
# Specify which files should be added to the gem when it is released.
|
21
21
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
data/bin/setup
CHANGED
@@ -11,21 +11,35 @@ bundle exec rails new spec/dummy --skip-git --skip-javascript --skip-hotwire --s
|
|
11
11
|
|
12
12
|
echo "Changes to application"
|
13
13
|
|
14
|
-
sed -i 's/
|
14
|
+
sed -i 's/2.7.4/>= 2.7.4/' spec/dummy/Gemfile
|
15
15
|
sed -i '/end/i root "main#index"' spec/dummy/config/routes.rb
|
16
16
|
sed -i '/root "main#index"/i get "create_job", to: "main#create_job", as: "create_job"' spec/dummy/config/routes.rb
|
17
|
+
sed -i '/root "main#index"/i get "send_email", to: "main#send_email", as: "send_email"' spec/dummy/config/routes.rb
|
17
18
|
|
18
19
|
echo '
|
19
20
|
class MainController < ApplicationController
|
20
21
|
include ActionView::Helpers::UrlHelper
|
21
|
-
def index
|
22
|
-
render inline: "#{link_to HelloWorldJob.name,create_job_path}"
|
23
|
-
end
|
24
22
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
23
|
+
def index
|
24
|
+
render inline: "#{link_to HelloWorldJob.name, create_job_path}<br>
|
25
|
+
#{link_to HelloWorldJob.name, create_job_path(wait: 1)}<br>
|
26
|
+
#{link_to SubscriptionMailer, send_email_path}"
|
27
|
+
end
|
28
|
+
|
29
|
+
def create_job
|
30
|
+
j = HelloWorldJob
|
31
|
+
if params[:wait]
|
32
|
+
j = j.set(wait: params[:wait].to_i.minutes)
|
33
|
+
end
|
34
|
+
j.perform_later
|
35
|
+
|
36
|
+
redirect_to root_path
|
37
|
+
end
|
38
|
+
|
39
|
+
def send_email
|
40
|
+
SubscriptionMailer.notify.deliver_later
|
41
|
+
redirect_to root_path
|
42
|
+
end
|
29
43
|
|
30
44
|
end
|
31
45
|
' > spec/dummy/app/controllers/main_controller.rb
|
@@ -52,8 +66,46 @@ Rails.application.configure do
|
|
52
66
|
namespace
|
53
67
|
)
|
54
68
|
|
69
|
+
default_manifest = YAML.safe_load(
|
70
|
+
<<~MANIFEST
|
71
|
+
apiVersion: batch/v1
|
72
|
+
kind: Job
|
73
|
+
metadata:
|
74
|
+
name: scheduled-job-name
|
75
|
+
namespace: default
|
76
|
+
spec:
|
77
|
+
template:
|
78
|
+
spec:
|
79
|
+
restartPolicy: Never
|
80
|
+
containers:
|
81
|
+
- name: app-job
|
82
|
+
image: activejobk8s:0.1.0
|
83
|
+
imagePullPolicy: IfNotPresent
|
84
|
+
env:
|
85
|
+
- name: RAILS_SERVE_STATIC_FILES
|
86
|
+
value: "1"
|
87
|
+
- name: RAILS_LOG_TO_STDOUT
|
88
|
+
value: "1"
|
89
|
+
- name: RAILS_ENV
|
90
|
+
value: "development"
|
91
|
+
command: ["/bin/sh"]
|
92
|
+
args:
|
93
|
+
- "-ec"
|
94
|
+
- "bundle install && bundle exec rails active_job_k8s:run_job"
|
95
|
+
volumeMounts:
|
96
|
+
- mountPath: /usr/src/app
|
97
|
+
name: application-volume
|
98
|
+
volumes:
|
99
|
+
- name: application-volume
|
100
|
+
hostPath:
|
101
|
+
# directory location on host
|
102
|
+
path: /application
|
103
|
+
MANIFEST
|
104
|
+
)
|
105
|
+
|
55
106
|
config.active_job.queue_adapter = ActiveJob::QueueAdapters::K8sAdapter.new(
|
56
|
-
kubeclient_context: context
|
107
|
+
kubeclient_context: context,
|
108
|
+
default_manifest: default_manifest
|
57
109
|
)
|
58
110
|
|
59
111
|
end
|
@@ -67,48 +119,16 @@ class HelloWorldJob < ApplicationJob
|
|
67
119
|
Rails.logger.debug "Hello World"
|
68
120
|
FileUtils.touch(Rails.root.join("tmp","#{SecureRandom.hex}.test"))
|
69
121
|
end
|
122
|
+
end
|
123
|
+
' > spec/dummy/app/jobs/hello_world_job.rb
|
70
124
|
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
kind: Job
|
76
|
-
metadata:
|
77
|
-
name: scheduled-job-name
|
78
|
-
namespace: default
|
79
|
-
spec:
|
80
|
-
template:
|
81
|
-
spec:
|
82
|
-
restartPolicy: Never
|
83
|
-
containers:
|
84
|
-
- name: app-job
|
85
|
-
image: activejobk8s:0.1.0
|
86
|
-
imagePullPolicy: IfNotPresent
|
87
|
-
env:
|
88
|
-
- name: RAILS_SERVE_STATIC_FILES
|
89
|
-
value: "1"
|
90
|
-
- name: RAILS_LOG_TO_STDOUT
|
91
|
-
value: "1"
|
92
|
-
- name: RAILS_ENV
|
93
|
-
value: "development"
|
94
|
-
command: ["/bin/sh"]
|
95
|
-
args:
|
96
|
-
- "-ec"
|
97
|
-
- "bundle install && bundle exec rails active_job_k8s:run_job"
|
98
|
-
volumeMounts:
|
99
|
-
- mountPath: /usr/src/app
|
100
|
-
name: application-volume
|
101
|
-
volumes:
|
102
|
-
- name: application-volume
|
103
|
-
hostPath:
|
104
|
-
# directory location on host
|
105
|
-
path: /application
|
106
|
-
|
107
|
-
MANIFEST
|
108
|
-
)
|
125
|
+
echo '
|
126
|
+
class SubscriptionMailer < ApplicationMailer
|
127
|
+
def notify
|
128
|
+
mail(to: "test@domain.tld", subject: "test", body: "message")
|
109
129
|
end
|
110
130
|
end
|
111
|
-
'
|
131
|
+
' > spec/dummy/app/mailers/subscription_mailer.rb
|
112
132
|
|
113
133
|
echo 'FROM ruby:3.0.4-alpine3.16
|
114
134
|
|
@@ -16,12 +16,10 @@ module ActiveJob
|
|
16
16
|
scheduler.create_job(job)
|
17
17
|
end
|
18
18
|
|
19
|
-
def enqueue_at(
|
20
|
-
|
21
|
-
raise NotImplementedError, "It will be implemented in the future. Read the roadmap"
|
19
|
+
def enqueue_at(job,scheduled_at)
|
20
|
+
scheduler.create_job(job,scheduled_at:scheduled_at)
|
22
21
|
end
|
23
22
|
|
24
|
-
|
25
23
|
end
|
26
24
|
end
|
27
25
|
end
|
@@ -5,5 +5,17 @@ module ActiveJobK8s
|
|
5
5
|
rake_tasks do
|
6
6
|
require 'active_job_k8s/tasks'
|
7
7
|
end
|
8
|
+
|
9
|
+
server do
|
10
|
+
puts "=> START ActiveJobK8s - ControlLoop sleep [5]"
|
11
|
+
Thread.new do
|
12
|
+
Rails.application.reloader.wrap do
|
13
|
+
loop do
|
14
|
+
Rails.application.config.active_job.queue_adapter.scheduler.un_suspend_jobs
|
15
|
+
sleep 5
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
8
20
|
end
|
9
21
|
end
|
@@ -5,26 +5,32 @@ require "json"
|
|
5
5
|
module ActiveJobK8s
|
6
6
|
class Scheduler
|
7
7
|
|
8
|
+
#@return [Kubeclient::Config::Context]
|
8
9
|
attr_reader :kubeclient_context
|
9
10
|
|
10
|
-
|
11
|
+
#@return [Hash]
|
12
|
+
attr_reader :default_manifest
|
13
|
+
|
14
|
+
# @param [Hash{ kubeclient_context: Kubeclient::Config::Context, default_manifest: Hash }] opts
|
11
15
|
def initialize(**opts)
|
12
16
|
raise "No KubeClientContext given" if opts[:kubeclient_context].nil?
|
13
17
|
# or to use a specific context, by name:
|
14
18
|
@kubeclient_context = opts[:kubeclient_context]
|
19
|
+
@default_manifest = opts[:default_manifest] || {}
|
15
20
|
|
16
21
|
end
|
17
22
|
|
18
|
-
def create_job(job)
|
23
|
+
def create_job(job, scheduled_at: nil)
|
19
24
|
|
20
25
|
serialized_job = JSON.dump(job.serialize)
|
21
26
|
|
22
|
-
|
27
|
+
manifest = (job.respond_to?(:manifest) and job.manifest.is_a?(Hash) and !job.manifest.empty?) ? job.manifest : default_manifest
|
28
|
+
kube_job = Kubeclient::Resource.new(manifest)
|
23
29
|
|
24
|
-
# kube_job.spec.suspend = false FIXME complete for delayed jobs
|
25
30
|
kube_job.metadata.name = "#{kube_job.metadata.name}-#{job.job_id}"
|
26
|
-
kube_job.metadata.
|
27
|
-
kube_job.metadata.
|
31
|
+
kube_job.metadata.annotations ||= {}
|
32
|
+
kube_job.metadata.annotations.job_id = job.job_id
|
33
|
+
kube_job.metadata.annotations.queue_name = job.queue_name
|
28
34
|
kube_job.metadata.namespace = kube_job.metadata.namespace || kubeclient_context.namespace
|
29
35
|
kube_job.spec.template.spec.containers.map do |container|
|
30
36
|
container.env ||= []
|
@@ -40,6 +46,14 @@ module ActiveJobK8s
|
|
40
46
|
end
|
41
47
|
kube_job.spec.ttlSecondsAfterFinished = 300 #number of seconds the job will be erased
|
42
48
|
|
49
|
+
kube_job.metadata.labels ||= {}
|
50
|
+
if scheduled_at
|
51
|
+
kube_job.spec.suspend = true
|
52
|
+
kube_job.metadata.annotations.scheduled_at = scheduled_at.to_s
|
53
|
+
kube_job.metadata.labels.activeJobK8s = "scheduled" # job to be execute when time comes
|
54
|
+
else
|
55
|
+
kube_job.metadata.labels.activeJobK8s = "delayed" # job to be execute when possible
|
56
|
+
end
|
43
57
|
client.create_job(kube_job)
|
44
58
|
end
|
45
59
|
|
@@ -47,6 +61,17 @@ module ActiveJobK8s
|
|
47
61
|
ActiveJob::Base.execute(JSON.parse(ENV['SERIALIZED_JOB']))
|
48
62
|
end
|
49
63
|
|
64
|
+
##
|
65
|
+
# Un-suspend jobs if the scheduled at is outdated
|
66
|
+
def un_suspend_jobs
|
67
|
+
suspended_jobs.each do |sj|
|
68
|
+
scheduled_at = Time.at(sj.metadata.annotations.scheduled_at.to_f)
|
69
|
+
if Time.now > scheduled_at and sj.spec.suspend
|
70
|
+
client.patch_job(sj.metadata.name, { spec: { suspend: false } }, sj.metadata.namespace).inspect
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
50
75
|
protected
|
51
76
|
|
52
77
|
def client
|
@@ -55,6 +80,16 @@ module ActiveJobK8s
|
|
55
80
|
ssl_options: @kubeclient_context.ssl_options,
|
56
81
|
auth_options: @kubeclient_context.auth_options)
|
57
82
|
end
|
83
|
+
|
84
|
+
private
|
85
|
+
|
86
|
+
##
|
87
|
+
# Internal list of all suspended jobs
|
88
|
+
def suspended_jobs
|
89
|
+
client.get_jobs(namespace: kubeclient_context.namespace,
|
90
|
+
label_selector: "activeJobK8s=scheduled",
|
91
|
+
field_selector: 'status.successful!=1')
|
92
|
+
end
|
58
93
|
end
|
59
94
|
|
60
95
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_job_k8s
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Marino Bonetti
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-11-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: kubeclient
|
@@ -79,7 +79,7 @@ metadata:
|
|
79
79
|
allowed_push_host: https://rubygems.org
|
80
80
|
homepage_uri: https://github.com/oniram88/active_job_k8s
|
81
81
|
source_code_uri: https://github.com/oniram88/active_job_k8s
|
82
|
-
changelog_uri: https://github.com/oniram88/active_job_k8s/CHANGELOG.md
|
82
|
+
changelog_uri: https://github.com/oniram88/active_job_k8s/blob/master/CHANGELOG.md
|
83
83
|
post_install_message:
|
84
84
|
rdoc_options: []
|
85
85
|
require_paths:
|
@@ -88,7 +88,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
88
88
|
requirements:
|
89
89
|
- - ">="
|
90
90
|
- !ruby/object:Gem::Version
|
91
|
-
version: 2.
|
91
|
+
version: 2.6.0
|
92
92
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
94
|
- - ">="
|