ood_core 0.15.1 → 0.16.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -1
- data/lib/ood_core/job/adapters/kubernetes/batch.rb +26 -8
- data/lib/ood_core/job/adapters/kubernetes/helper.rb +12 -7
- data/lib/ood_core/job/adapters/kubernetes/resources.rb +33 -9
- data/lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb +44 -14
- data/lib/ood_core/job/adapters/linux_host/launcher.rb +16 -9
- data/lib/ood_core/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 392bbf30609185792c59bc9b83bad0d2d6861885db4264a4da39356cb64624b6
|
4
|
+
data.tar.gz: 76df75234a646147a8fed14367d6467ffabe289bfa8264ff02614b60f6463652
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fd127307d7048a220bdd5bed28302e90708bb0a32c9c6905136ddb3664b8f0c5035510c528f255161c4d40c98a772d6200fa60c938aff10ea4a157fdf56d8d1f
|
7
|
+
data.tar.gz: f3c1ea0622da5b4387ee123843da2e0e955afd40d28e0f25e425fea010865896cdebf87171e2264ffbffbd743be8d1533971a25566251dcb8da5748191dcbeea
|
data/CHANGELOG.md
CHANGED
@@ -6,6 +6,24 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
|
6
6
|
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
7
7
|
|
8
8
|
## [Unreleased]
|
9
|
+
## [0.16.0] - 2021-04-20
|
10
|
+
### Fixed
|
11
|
+
- tmux 2.7+ bug in the linux host adapter in [2.5.8](https://github.com/OSC/ood_core/pull/258)
|
12
|
+
and [259](https://github.com/OSC/ood_core/pull/259).
|
13
|
+
|
14
|
+
### Changed
|
15
|
+
|
16
|
+
- Changed how k8s configmaps in are defined in [251](https://github.com/OSC/ood_core/pull/251).
|
17
|
+
The data structure now expects a key called files which is an array of objects that hold
|
18
|
+
filename, data, mount_path, sub_path and init_mount_path.
|
19
|
+
[255](https://github.com/OSC/ood_core/pull/255) also relates to this interface change.
|
20
|
+
|
21
|
+
### Added
|
22
|
+
|
23
|
+
- The k8s adapter can now specify environment variables and creates defaults
|
24
|
+
in [252](https://github.com/OSC/ood_core/pull/252).
|
25
|
+
- The k8s adapter can now specify image pull secrets in [253](https://github.com/OSC/ood_core/pull/253).
|
26
|
+
|
9
27
|
## [0.15.1] - 2021-02-25
|
10
28
|
### Fixed
|
11
29
|
- kubernetes adapter uses the full module for helpers in [245](https://github.com/OSC/ood_core/pull/245).
|
@@ -282,7 +300,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
|
|
282
300
|
### Added
|
283
301
|
- Initial release!
|
284
302
|
|
285
|
-
[Unreleased]: https://github.com/OSC/ood_core/compare/v0.
|
303
|
+
[Unreleased]: https://github.com/OSC/ood_core/compare/v0.16.0...HEAD
|
304
|
+
[0.15.2]: https://github.com/OSC/ood_core/compare/v0.15.1...v0.16.0
|
286
305
|
[0.15.1]: https://github.com/OSC/ood_core/compare/v0.15.0...v0.15.1
|
287
306
|
[0.15.0]: https://github.com/OSC/ood_core/compare/v0.14.0...v0.15.0
|
288
307
|
[0.14.0]: https://github.com/OSC/ood_core/compare/v0.13.0...v0.14.0
|
@@ -112,10 +112,6 @@ class OodCore::Job::Adapters::Kubernetes::Batch
|
|
112
112
|
safe_call("delete", "configmap", configmap_name(id))
|
113
113
|
end
|
114
114
|
|
115
|
-
def configmap_mount_path
|
116
|
-
'/ood'
|
117
|
-
end
|
118
|
-
|
119
115
|
private
|
120
116
|
|
121
117
|
def safe_call(verb, resource, id)
|
@@ -153,26 +149,48 @@ class OodCore::Job::Adapters::Kubernetes::Batch
|
|
153
149
|
username_prefix.nil? ? username : "#{username_prefix}-#{username}"
|
154
150
|
end
|
155
151
|
|
152
|
+
def user
|
153
|
+
@user ||= Etc.getpwnam(username)
|
154
|
+
end
|
155
|
+
|
156
|
+
def home_dir
|
157
|
+
user.dir
|
158
|
+
end
|
159
|
+
|
156
160
|
def run_as_user
|
157
|
-
|
161
|
+
user.uid
|
158
162
|
end
|
159
163
|
|
160
164
|
def run_as_group
|
161
|
-
|
165
|
+
user.gid
|
162
166
|
end
|
163
167
|
|
164
168
|
def fs_group
|
165
169
|
run_as_group
|
166
170
|
end
|
167
171
|
|
172
|
+
def group
|
173
|
+
Etc.getgrgid(run_as_group).name
|
174
|
+
end
|
175
|
+
|
176
|
+
def default_env
|
177
|
+
{
|
178
|
+
USER: username,
|
179
|
+
UID: run_as_user,
|
180
|
+
HOME: home_dir,
|
181
|
+
GROUP: group,
|
182
|
+
GID: run_as_group,
|
183
|
+
}
|
184
|
+
end
|
185
|
+
|
168
186
|
# helper to template resource yml you're going to submit and
|
169
187
|
# create an id.
|
170
188
|
def generate_id_yml(script)
|
171
189
|
native_data = script.native
|
172
|
-
container = helper.container_from_native(native_data[:container])
|
190
|
+
container = helper.container_from_native(native_data[:container], default_env)
|
173
191
|
id = generate_id(container.name)
|
174
192
|
configmap = helper.configmap_from_native(native_data, id)
|
175
|
-
init_containers = helper.init_ctrs_from_native(native_data[:init_containers])
|
193
|
+
init_containers = helper.init_ctrs_from_native(native_data[:init_containers], container.env)
|
176
194
|
spec = OodCore::Job::Adapters::Kubernetes::Resources::PodSpec.new(container, init_containers: init_containers)
|
177
195
|
all_mounts = native_data[:mounts].nil? ? mounts : mounts + native_data[:mounts]
|
178
196
|
|
@@ -38,18 +38,22 @@ class OodCore::Job::Adapters::Kubernetes::Helper
|
|
38
38
|
#
|
39
39
|
# @param container [#to_h]
|
40
40
|
# the input container hash
|
41
|
+
# @param default_env [#to_h]
|
42
|
+
# Default env to merge with defined env
|
41
43
|
# @return [OodCore::Job::Adapters::Kubernetes::Resources::Container]
|
42
|
-
def container_from_native(container)
|
44
|
+
def container_from_native(container, default_env)
|
45
|
+
env = container.fetch(:env, {}).to_h.symbolize_keys
|
43
46
|
OodCore::Job::Adapters::Kubernetes::Resources::Container.new(
|
44
47
|
container[:name],
|
45
48
|
container[:image],
|
46
49
|
command: parse_command(container[:command]),
|
47
50
|
port: container[:port],
|
48
|
-
env:
|
51
|
+
env: default_env.merge(env),
|
49
52
|
memory: container[:memory],
|
50
53
|
cpu: container[:cpu],
|
51
54
|
working_dir: container[:working_dir],
|
52
|
-
restart_policy: container[:restart_policy]
|
55
|
+
restart_policy: container[:restart_policy],
|
56
|
+
image_pull_secret: container[:image_pull_secret]
|
53
57
|
)
|
54
58
|
end
|
55
59
|
|
@@ -83,8 +87,7 @@ class OodCore::Job::Adapters::Kubernetes::Helper
|
|
83
87
|
|
84
88
|
OodCore::Job::Adapters::Kubernetes::Resources::ConfigMap.new(
|
85
89
|
configmap_name(id),
|
86
|
-
configmap[:
|
87
|
-
configmap[:data]
|
90
|
+
(configmap[:files] || [])
|
88
91
|
)
|
89
92
|
end
|
90
93
|
|
@@ -93,13 +96,15 @@ class OodCore::Job::Adapters::Kubernetes::Helper
|
|
93
96
|
# @param native_data [#to_h]
|
94
97
|
# the native data to parse. Expected key init_ctrs and for that
|
95
98
|
# key to be an array of hashes.
|
99
|
+
# @param default_env [#to_h]
|
100
|
+
# Default env to merge with defined env
|
96
101
|
# @return [Array<OodCore::Job::Adapters::Kubernetes::Resources::Container>]
|
97
102
|
# the array of init containers
|
98
|
-
def init_ctrs_from_native(ctrs)
|
103
|
+
def init_ctrs_from_native(ctrs, default_env)
|
99
104
|
init_ctrs = []
|
100
105
|
|
101
106
|
ctrs&.each do |ctr_raw|
|
102
|
-
ctr = container_from_native(ctr_raw)
|
107
|
+
ctr = container_from_native(ctr_raw, default_env)
|
103
108
|
init_ctrs.push(ctr)
|
104
109
|
end
|
105
110
|
|
@@ -1,22 +1,45 @@
|
|
1
1
|
module OodCore::Job::Adapters::Kubernetes::Resources
|
2
2
|
|
3
3
|
class ConfigMap
|
4
|
-
attr_accessor :name, :
|
4
|
+
attr_accessor :name, :files
|
5
5
|
|
6
|
-
def initialize(name,
|
6
|
+
def initialize(name, files)
|
7
7
|
@name = name
|
8
|
-
@
|
9
|
-
|
8
|
+
@files = []
|
9
|
+
files.each do |f|
|
10
|
+
@files << ConfigMapFile.new(f)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def mounts?
|
15
|
+
@files.any? { |f| f.mount_path }
|
16
|
+
end
|
17
|
+
|
18
|
+
def init_mounts?
|
19
|
+
@files.any? { |f| f.init_mount_path }
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
class ConfigMapFile
|
24
|
+
attr_accessor :filename, :data, :mount_path, :sub_path, :init_mount_path, :init_sub_path
|
25
|
+
|
26
|
+
def initialize(data)
|
27
|
+
@filename = data[:filename]
|
28
|
+
@data = data[:data]
|
29
|
+
@mount_path = data[:mount_path]
|
30
|
+
@sub_path = data[:sub_path]
|
31
|
+
@init_mount_path = data[:init_mount_path]
|
32
|
+
@init_sub_path = data[:init_sub_path]
|
10
33
|
end
|
11
34
|
end
|
12
35
|
|
13
36
|
class Container
|
14
37
|
attr_accessor :name, :image, :command, :port, :env, :memory, :cpu, :working_dir,
|
15
|
-
:restart_policy, :supplemental_groups
|
38
|
+
:restart_policy, :image_pull_secret, :supplemental_groups
|
16
39
|
|
17
40
|
def initialize(
|
18
|
-
name, image, command: [], port: nil, env:
|
19
|
-
working_dir: "", restart_policy: "Never", supplemental_groups: []
|
41
|
+
name, image, command: [], port: nil, env: {}, memory: "4Gi", cpu: "1",
|
42
|
+
working_dir: "", restart_policy: "Never", image_pull_secret: nil, supplemental_groups: []
|
20
43
|
)
|
21
44
|
raise ArgumentError, "containers need valid names and images" unless name && image
|
22
45
|
|
@@ -24,11 +47,12 @@ module OodCore::Job::Adapters::Kubernetes::Resources
|
|
24
47
|
@image = image
|
25
48
|
@command = command.nil? ? [] : command
|
26
49
|
@port = port&.to_i
|
27
|
-
@env = env.nil? ?
|
50
|
+
@env = env.nil? ? {} : env
|
28
51
|
@memory = memory.nil? ? "4Gi" : memory
|
29
52
|
@cpu = cpu.nil? ? "1" : cpu
|
30
53
|
@working_dir = working_dir.nil? ? "" : working_dir
|
31
54
|
@restart_policy = restart_policy.nil? ? "Never" : restart_policy
|
55
|
+
@image_pull_secret = image_pull_secret
|
32
56
|
@supplemental_groups = supplemental_groups.nil? ? [] : supplemental_groups
|
33
57
|
end
|
34
58
|
|
@@ -42,9 +66,9 @@ module OodCore::Job::Adapters::Kubernetes::Resources
|
|
42
66
|
cpu == other.cpu &&
|
43
67
|
working_dir == other.working_dir &&
|
44
68
|
restart_policy == other.restart_policy &&
|
69
|
+
image_pull_secret == other.image_pull_secret &&
|
45
70
|
supplemental_groups == other.supplemental_groups
|
46
71
|
end
|
47
|
-
|
48
72
|
end
|
49
73
|
|
50
74
|
class PodSpec
|
@@ -32,6 +32,10 @@ spec:
|
|
32
32
|
hostNetwork: false
|
33
33
|
hostIPC: false
|
34
34
|
hostPID: false
|
35
|
+
<%- unless spec.container.image_pull_secret.nil? -%>
|
36
|
+
imagePullSecrets:
|
37
|
+
- name: <%= spec.container.image_pull_secret %>
|
38
|
+
<%- end -%>
|
35
39
|
containers:
|
36
40
|
- name: "<%= spec.container.name %>"
|
37
41
|
image: <%= spec.container.image %>
|
@@ -39,13 +43,15 @@ spec:
|
|
39
43
|
<%- unless spec.container.working_dir.empty? -%>
|
40
44
|
workingDir: "<%= spec.container.working_dir %>"
|
41
45
|
<%- end -%>
|
42
|
-
<%- unless spec.container.env.empty? -%>
|
43
46
|
env:
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
+
- name: POD_NAME
|
48
|
+
valueFrom:
|
49
|
+
fieldRef:
|
50
|
+
fieldPath: metadata.name
|
51
|
+
<%- spec.container.env.each_pair do |name, value| -%>
|
52
|
+
- name: <%= name %>
|
53
|
+
value: "<%= value %>"
|
47
54
|
<%- end # for each env -%>
|
48
|
-
<%- end # unless env is nil -%>
|
49
55
|
<%- unless spec.container.command.empty? -%>
|
50
56
|
command:
|
51
57
|
<%- spec.container.command.each do |cmd| -%>
|
@@ -56,15 +62,21 @@ spec:
|
|
56
62
|
ports:
|
57
63
|
- containerPort: <%= spec.container.port %>
|
58
64
|
<%- end -%>
|
65
|
+
<%- if configmap.mounts? || !all_mounts.empty? -%>
|
59
66
|
volumeMounts:
|
60
|
-
<%-
|
67
|
+
<%- configmap.files.each do |file| -%>
|
68
|
+
<%- next if file.mount_path.nil? -%>
|
61
69
|
- name: configmap-volume
|
62
|
-
mountPath:
|
63
|
-
|
70
|
+
mountPath: <%= file.mount_path %>
|
71
|
+
<%- unless file.sub_path.nil? -%>
|
72
|
+
subPath: <%= file.sub_path %>
|
73
|
+
<%- end # end unless file.sub_path.nil? -%>
|
74
|
+
<%- end # end configmap.files.each -%>
|
64
75
|
<%- all_mounts.each do |mount| -%>
|
65
76
|
- name: <%= mount[:name] %>
|
66
77
|
mountPath: <%= mount[:destination_path] %>
|
67
78
|
<%- end # for each mount -%>
|
79
|
+
<%- end # configmap mounts? and all_mounts not empty -%>
|
68
80
|
resources:
|
69
81
|
limits:
|
70
82
|
memory: "<%= spec.container.memory %>"
|
@@ -83,19 +95,34 @@ spec:
|
|
83
95
|
<%- spec.init_containers.each do |ctr| -%>
|
84
96
|
- name: "<%= ctr.name %>"
|
85
97
|
image: "<%= ctr.image %>"
|
98
|
+
env:
|
99
|
+
- name: POD_NAME
|
100
|
+
valueFrom:
|
101
|
+
fieldRef:
|
102
|
+
fieldPath: metadata.name
|
103
|
+
<%- ctr.env.each_pair do |name, value| -%>
|
104
|
+
- name: <%= name %>
|
105
|
+
value: "<%= value %>"
|
106
|
+
<%- end # for each env -%>
|
86
107
|
command:
|
87
108
|
<%- ctr.command.each do |cmd| -%>
|
88
109
|
- "<%= cmd %>"
|
89
110
|
<%- end # command loop -%>
|
111
|
+
<%- if configmap.init_mounts? || !all_mounts.empty? -%>
|
90
112
|
volumeMounts:
|
91
|
-
<%-
|
113
|
+
<%- configmap.files.each do |file| -%>
|
114
|
+
<%- next if file.init_mount_path.nil? -%>
|
92
115
|
- name: configmap-volume
|
93
|
-
mountPath:
|
94
|
-
|
116
|
+
mountPath: <%= file.init_mount_path %>
|
117
|
+
<%- unless file.init_sub_path.nil? -%>
|
118
|
+
subPath: <%= file.init_sub_path %>
|
119
|
+
<%- end # end unless file.sub_path.nil? -%>
|
120
|
+
<%- end # end configmap.files.each -%>
|
95
121
|
<%- all_mounts.each do |mount| -%>
|
96
122
|
- name: <%= mount[:name] %>
|
97
123
|
mountPath: <%= mount[:destination_path] %>
|
98
124
|
<%- end # for each mount -%>
|
125
|
+
<%- end # if config_map init mounts and all_mounts not empty -%>
|
99
126
|
securityContext:
|
100
127
|
allowPrivilegeEscalation: false
|
101
128
|
capabilities:
|
@@ -153,6 +180,9 @@ metadata:
|
|
153
180
|
labels:
|
154
181
|
job: <%= id %>
|
155
182
|
data:
|
156
|
-
|
157
|
-
|
158
|
-
|
183
|
+
<%- configmap.files.each do |file| -%>
|
184
|
+
<%- next if file.data.nil? || file.filename.nil? -%>
|
185
|
+
<%= file.filename %>: |
|
186
|
+
<% config_data_lines(file.data).each do |line| %><%= line %><% end %>
|
187
|
+
<%- end # end for configmap files -%>
|
188
|
+
<%- end # end configmap.nil? %>
|
@@ -16,7 +16,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
|
|
16
16
|
# from
|
17
17
|
class Error < StandardError; end
|
18
18
|
|
19
|
-
UNIT_SEPARATOR = "
|
19
|
+
UNIT_SEPARATOR = ","
|
20
20
|
|
21
21
|
# @param debug Whether the adapter should be used in debug mode
|
22
22
|
# @param site_timeout [#to_i] A period after which the job should be killed or nil
|
@@ -80,12 +80,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
|
|
80
80
|
|
81
81
|
call(*cmd, stdin: kill_cmd)
|
82
82
|
rescue Error => e
|
83
|
-
|
84
|
-
# The tmux server not running is not an error
|
85
|
-
e.message.include?('failed to connect to server') ||
|
86
|
-
# The session not being found is not an error
|
87
|
-
e.message.include?("session not found: #{session_name_label}")
|
88
|
-
)
|
83
|
+
interpret_and_raise(e)
|
89
84
|
end
|
90
85
|
|
91
86
|
def list_remote_sessions(host: nil)
|
@@ -264,8 +259,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
|
|
264
259
|
|session_hash| session_hash[:session_name].start_with?(session_name_label)
|
265
260
|
}
|
266
261
|
rescue Error => e
|
267
|
-
|
268
|
-
raise e unless e.message.include?('failed to connect to server')
|
262
|
+
interpret_and_raise(e)
|
269
263
|
[]
|
270
264
|
end
|
271
265
|
|
@@ -280,4 +274,17 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
|
|
280
274
|
|
281
275
|
'/dev/null'
|
282
276
|
end
|
277
|
+
|
278
|
+
# under some conditions tmux returns status code 1 but it's not an actual
|
279
|
+
# error. These are when the session is not found or there are no sessions
|
280
|
+
# at all.
|
281
|
+
def interpret_and_raise(error)
|
282
|
+
if error.message.include?('failed to connect to server') # no sessions in tmux 1.8
|
283
|
+
nil
|
284
|
+
elsif error.message.include?('no server running on') # no sessions in tmux 2.7+ message
|
285
|
+
nil
|
286
|
+
else
|
287
|
+
raise error
|
288
|
+
end
|
289
|
+
end
|
283
290
|
end
|
data/lib/ood_core/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ood_core
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.16.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Eric Franz
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: exe
|
12
12
|
cert_chain: []
|
13
|
-
date: 2021-
|
13
|
+
date: 2021-04-20 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: ood_support
|