ood_core 0.15.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -1
- data/lib/ood_core/job/adapters/kubernetes/batch.rb +26 -8
- data/lib/ood_core/job/adapters/kubernetes/helper.rb +12 -7
- data/lib/ood_core/job/adapters/kubernetes/resources.rb +33 -9
- data/lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb +44 -14
- data/lib/ood_core/job/adapters/linux_host/launcher.rb +16 -9
- data/lib/ood_core/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 392bbf30609185792c59bc9b83bad0d2d6861885db4264a4da39356cb64624b6
|
4
|
+
data.tar.gz: 76df75234a646147a8fed14367d6467ffabe289bfa8264ff02614b60f6463652
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fd127307d7048a220bdd5bed28302e90708bb0a32c9c6905136ddb3664b8f0c5035510c528f255161c4d40c98a772d6200fa60c938aff10ea4a157fdf56d8d1f
|
7
|
+
data.tar.gz: f3c1ea0622da5b4387ee123843da2e0e955afd40d28e0f25e425fea010865896cdebf87171e2264ffbffbd743be8d1533971a25566251dcb8da5748191dcbeea
|
data/CHANGELOG.md
CHANGED
@@ -6,6 +6,24 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
|
|
6
6
|
and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
|
7
7
|
|
8
8
|
## [Unreleased]
|
9
|
+
## [0.16.0] - 2021-04-20
|
10
|
+
### Fixed
|
11
|
+
- tmux 2.7+ bug in the linux host adapter in [2.5.8](https://github.com/OSC/ood_core/pull/258)
|
12
|
+
and [259](https://github.com/OSC/ood_core/pull/259).
|
13
|
+
|
14
|
+
### Changed
|
15
|
+
|
16
|
+
- Changed how k8s configmaps in are defined in [251](https://github.com/OSC/ood_core/pull/251).
|
17
|
+
The data structure now expects a key called files which is an array of objects that hold
|
18
|
+
filename, data, mount_path, sub_path and init_mount_path.
|
19
|
+
[255](https://github.com/OSC/ood_core/pull/255) also relates to this interface change.
|
20
|
+
|
21
|
+
### Added
|
22
|
+
|
23
|
+
- The k8s adapter can now specify environment variables and creates defaults
|
24
|
+
in [252](https://github.com/OSC/ood_core/pull/252).
|
25
|
+
- The k8s adapter can now specify image pull secrets in [253](https://github.com/OSC/ood_core/pull/253).
|
26
|
+
|
9
27
|
## [0.15.1] - 2021-02-25
|
10
28
|
### Fixed
|
11
29
|
- kubernetes adapter uses the full module for helpers in [245](https://github.com/OSC/ood_core/pull/245).
|
@@ -282,7 +300,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
|
|
282
300
|
### Added
|
283
301
|
- Initial release!
|
284
302
|
|
285
|
-
[Unreleased]: https://github.com/OSC/ood_core/compare/v0.
|
303
|
+
[Unreleased]: https://github.com/OSC/ood_core/compare/v0.16.0...HEAD
|
304
|
+
[0.15.2]: https://github.com/OSC/ood_core/compare/v0.15.1...v0.16.0
|
286
305
|
[0.15.1]: https://github.com/OSC/ood_core/compare/v0.15.0...v0.15.1
|
287
306
|
[0.15.0]: https://github.com/OSC/ood_core/compare/v0.14.0...v0.15.0
|
288
307
|
[0.14.0]: https://github.com/OSC/ood_core/compare/v0.13.0...v0.14.0
|
@@ -112,10 +112,6 @@ class OodCore::Job::Adapters::Kubernetes::Batch
|
|
112
112
|
safe_call("delete", "configmap", configmap_name(id))
|
113
113
|
end
|
114
114
|
|
115
|
-
def configmap_mount_path
|
116
|
-
'/ood'
|
117
|
-
end
|
118
|
-
|
119
115
|
private
|
120
116
|
|
121
117
|
def safe_call(verb, resource, id)
|
@@ -153,26 +149,48 @@ class OodCore::Job::Adapters::Kubernetes::Batch
|
|
153
149
|
username_prefix.nil? ? username : "#{username_prefix}-#{username}"
|
154
150
|
end
|
155
151
|
|
152
|
+
def user
|
153
|
+
@user ||= Etc.getpwnam(username)
|
154
|
+
end
|
155
|
+
|
156
|
+
def home_dir
|
157
|
+
user.dir
|
158
|
+
end
|
159
|
+
|
156
160
|
def run_as_user
|
157
|
-
|
161
|
+
user.uid
|
158
162
|
end
|
159
163
|
|
160
164
|
def run_as_group
|
161
|
-
|
165
|
+
user.gid
|
162
166
|
end
|
163
167
|
|
164
168
|
def fs_group
|
165
169
|
run_as_group
|
166
170
|
end
|
167
171
|
|
172
|
+
def group
|
173
|
+
Etc.getgrgid(run_as_group).name
|
174
|
+
end
|
175
|
+
|
176
|
+
def default_env
|
177
|
+
{
|
178
|
+
USER: username,
|
179
|
+
UID: run_as_user,
|
180
|
+
HOME: home_dir,
|
181
|
+
GROUP: group,
|
182
|
+
GID: run_as_group,
|
183
|
+
}
|
184
|
+
end
|
185
|
+
|
168
186
|
# helper to template resource yml you're going to submit and
|
169
187
|
# create an id.
|
170
188
|
def generate_id_yml(script)
|
171
189
|
native_data = script.native
|
172
|
-
container = helper.container_from_native(native_data[:container])
|
190
|
+
container = helper.container_from_native(native_data[:container], default_env)
|
173
191
|
id = generate_id(container.name)
|
174
192
|
configmap = helper.configmap_from_native(native_data, id)
|
175
|
-
init_containers = helper.init_ctrs_from_native(native_data[:init_containers])
|
193
|
+
init_containers = helper.init_ctrs_from_native(native_data[:init_containers], container.env)
|
176
194
|
spec = OodCore::Job::Adapters::Kubernetes::Resources::PodSpec.new(container, init_containers: init_containers)
|
177
195
|
all_mounts = native_data[:mounts].nil? ? mounts : mounts + native_data[:mounts]
|
178
196
|
|
@@ -38,18 +38,22 @@ class OodCore::Job::Adapters::Kubernetes::Helper
|
|
38
38
|
#
|
39
39
|
# @param container [#to_h]
|
40
40
|
# the input container hash
|
41
|
+
# @param default_env [#to_h]
|
42
|
+
# Default env to merge with defined env
|
41
43
|
# @return [OodCore::Job::Adapters::Kubernetes::Resources::Container]
|
42
|
-
def container_from_native(container)
|
44
|
+
def container_from_native(container, default_env)
|
45
|
+
env = container.fetch(:env, {}).to_h.symbolize_keys
|
43
46
|
OodCore::Job::Adapters::Kubernetes::Resources::Container.new(
|
44
47
|
container[:name],
|
45
48
|
container[:image],
|
46
49
|
command: parse_command(container[:command]),
|
47
50
|
port: container[:port],
|
48
|
-
env:
|
51
|
+
env: default_env.merge(env),
|
49
52
|
memory: container[:memory],
|
50
53
|
cpu: container[:cpu],
|
51
54
|
working_dir: container[:working_dir],
|
52
|
-
restart_policy: container[:restart_policy]
|
55
|
+
restart_policy: container[:restart_policy],
|
56
|
+
image_pull_secret: container[:image_pull_secret]
|
53
57
|
)
|
54
58
|
end
|
55
59
|
|
@@ -83,8 +87,7 @@ class OodCore::Job::Adapters::Kubernetes::Helper
|
|
83
87
|
|
84
88
|
OodCore::Job::Adapters::Kubernetes::Resources::ConfigMap.new(
|
85
89
|
configmap_name(id),
|
86
|
-
configmap[:
|
87
|
-
configmap[:data]
|
90
|
+
(configmap[:files] || [])
|
88
91
|
)
|
89
92
|
end
|
90
93
|
|
@@ -93,13 +96,15 @@ class OodCore::Job::Adapters::Kubernetes::Helper
|
|
93
96
|
# @param native_data [#to_h]
|
94
97
|
# the native data to parse. Expected key init_ctrs and for that
|
95
98
|
# key to be an array of hashes.
|
99
|
+
# @param default_env [#to_h]
|
100
|
+
# Default env to merge with defined env
|
96
101
|
# @return [Array<OodCore::Job::Adapters::Kubernetes::Resources::Container>]
|
97
102
|
# the array of init containers
|
98
|
-
def init_ctrs_from_native(ctrs)
|
103
|
+
def init_ctrs_from_native(ctrs, default_env)
|
99
104
|
init_ctrs = []
|
100
105
|
|
101
106
|
ctrs&.each do |ctr_raw|
|
102
|
-
ctr = container_from_native(ctr_raw)
|
107
|
+
ctr = container_from_native(ctr_raw, default_env)
|
103
108
|
init_ctrs.push(ctr)
|
104
109
|
end
|
105
110
|
|
@@ -1,22 +1,45 @@
|
|
1
1
|
module OodCore::Job::Adapters::Kubernetes::Resources
|
2
2
|
|
3
3
|
class ConfigMap
|
4
|
-
attr_accessor :name, :
|
4
|
+
attr_accessor :name, :files
|
5
5
|
|
6
|
-
def initialize(name,
|
6
|
+
def initialize(name, files)
|
7
7
|
@name = name
|
8
|
-
@
|
9
|
-
|
8
|
+
@files = []
|
9
|
+
files.each do |f|
|
10
|
+
@files << ConfigMapFile.new(f)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def mounts?
|
15
|
+
@files.any? { |f| f.mount_path }
|
16
|
+
end
|
17
|
+
|
18
|
+
def init_mounts?
|
19
|
+
@files.any? { |f| f.init_mount_path }
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
class ConfigMapFile
|
24
|
+
attr_accessor :filename, :data, :mount_path, :sub_path, :init_mount_path, :init_sub_path
|
25
|
+
|
26
|
+
def initialize(data)
|
27
|
+
@filename = data[:filename]
|
28
|
+
@data = data[:data]
|
29
|
+
@mount_path = data[:mount_path]
|
30
|
+
@sub_path = data[:sub_path]
|
31
|
+
@init_mount_path = data[:init_mount_path]
|
32
|
+
@init_sub_path = data[:init_sub_path]
|
10
33
|
end
|
11
34
|
end
|
12
35
|
|
13
36
|
class Container
|
14
37
|
attr_accessor :name, :image, :command, :port, :env, :memory, :cpu, :working_dir,
|
15
|
-
:restart_policy, :supplemental_groups
|
38
|
+
:restart_policy, :image_pull_secret, :supplemental_groups
|
16
39
|
|
17
40
|
def initialize(
|
18
|
-
name, image, command: [], port: nil, env:
|
19
|
-
working_dir: "", restart_policy: "Never", supplemental_groups: []
|
41
|
+
name, image, command: [], port: nil, env: {}, memory: "4Gi", cpu: "1",
|
42
|
+
working_dir: "", restart_policy: "Never", image_pull_secret: nil, supplemental_groups: []
|
20
43
|
)
|
21
44
|
raise ArgumentError, "containers need valid names and images" unless name && image
|
22
45
|
|
@@ -24,11 +47,12 @@ module OodCore::Job::Adapters::Kubernetes::Resources
|
|
24
47
|
@image = image
|
25
48
|
@command = command.nil? ? [] : command
|
26
49
|
@port = port&.to_i
|
27
|
-
@env = env.nil? ?
|
50
|
+
@env = env.nil? ? {} : env
|
28
51
|
@memory = memory.nil? ? "4Gi" : memory
|
29
52
|
@cpu = cpu.nil? ? "1" : cpu
|
30
53
|
@working_dir = working_dir.nil? ? "" : working_dir
|
31
54
|
@restart_policy = restart_policy.nil? ? "Never" : restart_policy
|
55
|
+
@image_pull_secret = image_pull_secret
|
32
56
|
@supplemental_groups = supplemental_groups.nil? ? [] : supplemental_groups
|
33
57
|
end
|
34
58
|
|
@@ -42,9 +66,9 @@ module OodCore::Job::Adapters::Kubernetes::Resources
|
|
42
66
|
cpu == other.cpu &&
|
43
67
|
working_dir == other.working_dir &&
|
44
68
|
restart_policy == other.restart_policy &&
|
69
|
+
image_pull_secret == other.image_pull_secret &&
|
45
70
|
supplemental_groups == other.supplemental_groups
|
46
71
|
end
|
47
|
-
|
48
72
|
end
|
49
73
|
|
50
74
|
class PodSpec
|
@@ -32,6 +32,10 @@ spec:
|
|
32
32
|
hostNetwork: false
|
33
33
|
hostIPC: false
|
34
34
|
hostPID: false
|
35
|
+
<%- unless spec.container.image_pull_secret.nil? -%>
|
36
|
+
imagePullSecrets:
|
37
|
+
- name: <%= spec.container.image_pull_secret %>
|
38
|
+
<%- end -%>
|
35
39
|
containers:
|
36
40
|
- name: "<%= spec.container.name %>"
|
37
41
|
image: <%= spec.container.image %>
|
@@ -39,13 +43,15 @@ spec:
|
|
39
43
|
<%- unless spec.container.working_dir.empty? -%>
|
40
44
|
workingDir: "<%= spec.container.working_dir %>"
|
41
45
|
<%- end -%>
|
42
|
-
<%- unless spec.container.env.empty? -%>
|
43
46
|
env:
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
+
- name: POD_NAME
|
48
|
+
valueFrom:
|
49
|
+
fieldRef:
|
50
|
+
fieldPath: metadata.name
|
51
|
+
<%- spec.container.env.each_pair do |name, value| -%>
|
52
|
+
- name: <%= name %>
|
53
|
+
value: "<%= value %>"
|
47
54
|
<%- end # for each env -%>
|
48
|
-
<%- end # unless env is nil -%>
|
49
55
|
<%- unless spec.container.command.empty? -%>
|
50
56
|
command:
|
51
57
|
<%- spec.container.command.each do |cmd| -%>
|
@@ -56,15 +62,21 @@ spec:
|
|
56
62
|
ports:
|
57
63
|
- containerPort: <%= spec.container.port %>
|
58
64
|
<%- end -%>
|
65
|
+
<%- if configmap.mounts? || !all_mounts.empty? -%>
|
59
66
|
volumeMounts:
|
60
|
-
<%-
|
67
|
+
<%- configmap.files.each do |file| -%>
|
68
|
+
<%- next if file.mount_path.nil? -%>
|
61
69
|
- name: configmap-volume
|
62
|
-
mountPath:
|
63
|
-
|
70
|
+
mountPath: <%= file.mount_path %>
|
71
|
+
<%- unless file.sub_path.nil? -%>
|
72
|
+
subPath: <%= file.sub_path %>
|
73
|
+
<%- end # end unless file.sub_path.nil? -%>
|
74
|
+
<%- end # end configmap.files.each -%>
|
64
75
|
<%- all_mounts.each do |mount| -%>
|
65
76
|
- name: <%= mount[:name] %>
|
66
77
|
mountPath: <%= mount[:destination_path] %>
|
67
78
|
<%- end # for each mount -%>
|
79
|
+
<%- end # configmap mounts? and all_mounts not empty -%>
|
68
80
|
resources:
|
69
81
|
limits:
|
70
82
|
memory: "<%= spec.container.memory %>"
|
@@ -83,19 +95,34 @@ spec:
|
|
83
95
|
<%- spec.init_containers.each do |ctr| -%>
|
84
96
|
- name: "<%= ctr.name %>"
|
85
97
|
image: "<%= ctr.image %>"
|
98
|
+
env:
|
99
|
+
- name: POD_NAME
|
100
|
+
valueFrom:
|
101
|
+
fieldRef:
|
102
|
+
fieldPath: metadata.name
|
103
|
+
<%- ctr.env.each_pair do |name, value| -%>
|
104
|
+
- name: <%= name %>
|
105
|
+
value: "<%= value %>"
|
106
|
+
<%- end # for each env -%>
|
86
107
|
command:
|
87
108
|
<%- ctr.command.each do |cmd| -%>
|
88
109
|
- "<%= cmd %>"
|
89
110
|
<%- end # command loop -%>
|
111
|
+
<%- if configmap.init_mounts? || !all_mounts.empty? -%>
|
90
112
|
volumeMounts:
|
91
|
-
<%-
|
113
|
+
<%- configmap.files.each do |file| -%>
|
114
|
+
<%- next if file.init_mount_path.nil? -%>
|
92
115
|
- name: configmap-volume
|
93
|
-
mountPath:
|
94
|
-
|
116
|
+
mountPath: <%= file.init_mount_path %>
|
117
|
+
<%- unless file.init_sub_path.nil? -%>
|
118
|
+
subPath: <%= file.init_sub_path %>
|
119
|
+
<%- end # end unless file.sub_path.nil? -%>
|
120
|
+
<%- end # end configmap.files.each -%>
|
95
121
|
<%- all_mounts.each do |mount| -%>
|
96
122
|
- name: <%= mount[:name] %>
|
97
123
|
mountPath: <%= mount[:destination_path] %>
|
98
124
|
<%- end # for each mount -%>
|
125
|
+
<%- end # if config_map init mounts and all_mounts not empty -%>
|
99
126
|
securityContext:
|
100
127
|
allowPrivilegeEscalation: false
|
101
128
|
capabilities:
|
@@ -153,6 +180,9 @@ metadata:
|
|
153
180
|
labels:
|
154
181
|
job: <%= id %>
|
155
182
|
data:
|
156
|
-
|
157
|
-
|
158
|
-
|
183
|
+
<%- configmap.files.each do |file| -%>
|
184
|
+
<%- next if file.data.nil? || file.filename.nil? -%>
|
185
|
+
<%= file.filename %>: |
|
186
|
+
<% config_data_lines(file.data).each do |line| %><%= line %><% end %>
|
187
|
+
<%- end # end for configmap files -%>
|
188
|
+
<%- end # end configmap.nil? %>
|
@@ -16,7 +16,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
|
|
16
16
|
# from
|
17
17
|
class Error < StandardError; end
|
18
18
|
|
19
|
-
UNIT_SEPARATOR = "
|
19
|
+
UNIT_SEPARATOR = ","
|
20
20
|
|
21
21
|
# @param debug Whether the adapter should be used in debug mode
|
22
22
|
# @param site_timeout [#to_i] A period after which the job should be killed or nil
|
@@ -80,12 +80,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
|
|
80
80
|
|
81
81
|
call(*cmd, stdin: kill_cmd)
|
82
82
|
rescue Error => e
|
83
|
-
|
84
|
-
# The tmux server not running is not an error
|
85
|
-
e.message.include?('failed to connect to server') ||
|
86
|
-
# The session not being found is not an error
|
87
|
-
e.message.include?("session not found: #{session_name_label}")
|
88
|
-
)
|
83
|
+
interpret_and_raise(e)
|
89
84
|
end
|
90
85
|
|
91
86
|
def list_remote_sessions(host: nil)
|
@@ -264,8 +259,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
|
|
264
259
|
|session_hash| session_hash[:session_name].start_with?(session_name_label)
|
265
260
|
}
|
266
261
|
rescue Error => e
|
267
|
-
|
268
|
-
raise e unless e.message.include?('failed to connect to server')
|
262
|
+
interpret_and_raise(e)
|
269
263
|
[]
|
270
264
|
end
|
271
265
|
|
@@ -280,4 +274,17 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
|
|
280
274
|
|
281
275
|
'/dev/null'
|
282
276
|
end
|
277
|
+
|
278
|
+
# under some conditions tmux returns status code 1 but it's not an actual
|
279
|
+
# error. These are when the session is not found or there are no sessions
|
280
|
+
# at all.
|
281
|
+
def interpret_and_raise(error)
|
282
|
+
if error.message.include?('failed to connect to server') # no sessions in tmux 1.8
|
283
|
+
nil
|
284
|
+
elsif error.message.include?('no server running on') # no sessions in tmux 2.7+ message
|
285
|
+
nil
|
286
|
+
else
|
287
|
+
raise error
|
288
|
+
end
|
289
|
+
end
|
283
290
|
end
|
data/lib/ood_core/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ood_core
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.16.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Eric Franz
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: exe
|
12
12
|
cert_chain: []
|
13
|
-
date: 2021-
|
13
|
+
date: 2021-04-20 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: ood_support
|