ood_core 0.15.1 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: fde32140cb148c6ea939a1d2308446e9144aad5c853fc8c41ea839beadedf03b
4
- data.tar.gz: 5925bb0f8576864a3e37696d1c5b32a258edac5ebf78d07a6d509f4ec77c2339
3
+ metadata.gz: 392bbf30609185792c59bc9b83bad0d2d6861885db4264a4da39356cb64624b6
4
+ data.tar.gz: 76df75234a646147a8fed14367d6467ffabe289bfa8264ff02614b60f6463652
5
5
  SHA512:
6
- metadata.gz: c9e1d0bd9e423af5289a445c12438875ae5d74b25295c8917209f19ec69b8e84bdb74b6f4ae2da3450a344398e8d96da4ae464498a38c9146d87fff1d1bbb2dd
7
- data.tar.gz: b8daebdca0ed93b8d2ebb9089657efbb2b5a88e0b78b76607090b7c5befb96fbaa4a51e820b0236dc596d76c81b8110d1d8b53090f38abbfe01e00d411c96cd5
6
+ metadata.gz: fd127307d7048a220bdd5bed28302e90708bb0a32c9c6905136ddb3664b8f0c5035510c528f255161c4d40c98a772d6200fa60c938aff10ea4a157fdf56d8d1f
7
+ data.tar.gz: f3c1ea0622da5b4387ee123843da2e0e955afd40d28e0f25e425fea010865896cdebf87171e2264ffbffbd743be8d1533971a25566251dcb8da5748191dcbeea
data/CHANGELOG.md CHANGED
@@ -6,6 +6,24 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
6
6
  and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
7
7
 
8
8
  ## [Unreleased]
9
+ ## [0.16.0] - 2021-04-20
10
+ ### Fixed
11
+ - tmux 2.7+ bug in the linux host adapter in [2.5.8](https://github.com/OSC/ood_core/pull/258)
12
+ and [259](https://github.com/OSC/ood_core/pull/259).
13
+
14
+ ### Changed
15
+
16
+ - Changed how k8s configmaps in are defined in [251](https://github.com/OSC/ood_core/pull/251).
17
+ The data structure now expects a key called files which is an array of objects that hold
18
+ filename, data, mount_path, sub_path and init_mount_path.
19
+ [255](https://github.com/OSC/ood_core/pull/255) also relates to this interface change.
20
+
21
+ ### Added
22
+
23
+ - The k8s adapter can now specify environment variables and creates defaults
24
+ in [252](https://github.com/OSC/ood_core/pull/252).
25
+ - The k8s adapter can now specify image pull secrets in [253](https://github.com/OSC/ood_core/pull/253).
26
+
9
27
  ## [0.15.1] - 2021-02-25
10
28
  ### Fixed
11
29
  - kubernetes adapter uses the full module for helpers in [245](https://github.com/OSC/ood_core/pull/245).
@@ -282,7 +300,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
282
300
  ### Added
283
301
  - Initial release!
284
302
 
285
- [Unreleased]: https://github.com/OSC/ood_core/compare/v0.15.1...HEAD
303
+ [Unreleased]: https://github.com/OSC/ood_core/compare/v0.16.0...HEAD
304
+ [0.15.2]: https://github.com/OSC/ood_core/compare/v0.15.1...v0.16.0
286
305
  [0.15.1]: https://github.com/OSC/ood_core/compare/v0.15.0...v0.15.1
287
306
  [0.15.0]: https://github.com/OSC/ood_core/compare/v0.14.0...v0.15.0
288
307
  [0.14.0]: https://github.com/OSC/ood_core/compare/v0.13.0...v0.14.0
@@ -112,10 +112,6 @@ class OodCore::Job::Adapters::Kubernetes::Batch
112
112
  safe_call("delete", "configmap", configmap_name(id))
113
113
  end
114
114
 
115
- def configmap_mount_path
116
- '/ood'
117
- end
118
-
119
115
  private
120
116
 
121
117
  def safe_call(verb, resource, id)
@@ -153,26 +149,48 @@ class OodCore::Job::Adapters::Kubernetes::Batch
153
149
  username_prefix.nil? ? username : "#{username_prefix}-#{username}"
154
150
  end
155
151
 
152
+ def user
153
+ @user ||= Etc.getpwnam(username)
154
+ end
155
+
156
+ def home_dir
157
+ user.dir
158
+ end
159
+
156
160
  def run_as_user
157
- Etc.getpwnam(username).uid
161
+ user.uid
158
162
  end
159
163
 
160
164
  def run_as_group
161
- Etc.getpwnam(username).gid
165
+ user.gid
162
166
  end
163
167
 
164
168
  def fs_group
165
169
  run_as_group
166
170
  end
167
171
 
172
+ def group
173
+ Etc.getgrgid(run_as_group).name
174
+ end
175
+
176
+ def default_env
177
+ {
178
+ USER: username,
179
+ UID: run_as_user,
180
+ HOME: home_dir,
181
+ GROUP: group,
182
+ GID: run_as_group,
183
+ }
184
+ end
185
+
168
186
  # helper to template resource yml you're going to submit and
169
187
  # create an id.
170
188
  def generate_id_yml(script)
171
189
  native_data = script.native
172
- container = helper.container_from_native(native_data[:container])
190
+ container = helper.container_from_native(native_data[:container], default_env)
173
191
  id = generate_id(container.name)
174
192
  configmap = helper.configmap_from_native(native_data, id)
175
- init_containers = helper.init_ctrs_from_native(native_data[:init_containers])
193
+ init_containers = helper.init_ctrs_from_native(native_data[:init_containers], container.env)
176
194
  spec = OodCore::Job::Adapters::Kubernetes::Resources::PodSpec.new(container, init_containers: init_containers)
177
195
  all_mounts = native_data[:mounts].nil? ? mounts : mounts + native_data[:mounts]
178
196
 
@@ -38,18 +38,22 @@ class OodCore::Job::Adapters::Kubernetes::Helper
38
38
  #
39
39
  # @param container [#to_h]
40
40
  # the input container hash
41
+ # @param default_env [#to_h]
42
+ # Default env to merge with defined env
41
43
  # @return [OodCore::Job::Adapters::Kubernetes::Resources::Container]
42
- def container_from_native(container)
44
+ def container_from_native(container, default_env)
45
+ env = container.fetch(:env, {}).to_h.symbolize_keys
43
46
  OodCore::Job::Adapters::Kubernetes::Resources::Container.new(
44
47
  container[:name],
45
48
  container[:image],
46
49
  command: parse_command(container[:command]),
47
50
  port: container[:port],
48
- env: container.fetch(:env, []),
51
+ env: default_env.merge(env),
49
52
  memory: container[:memory],
50
53
  cpu: container[:cpu],
51
54
  working_dir: container[:working_dir],
52
- restart_policy: container[:restart_policy]
55
+ restart_policy: container[:restart_policy],
56
+ image_pull_secret: container[:image_pull_secret]
53
57
  )
54
58
  end
55
59
 
@@ -83,8 +87,7 @@ class OodCore::Job::Adapters::Kubernetes::Helper
83
87
 
84
88
  OodCore::Job::Adapters::Kubernetes::Resources::ConfigMap.new(
85
89
  configmap_name(id),
86
- configmap[:filename],
87
- configmap[:data]
90
+ (configmap[:files] || [])
88
91
  )
89
92
  end
90
93
 
@@ -93,13 +96,15 @@ class OodCore::Job::Adapters::Kubernetes::Helper
93
96
  # @param native_data [#to_h]
94
97
  # the native data to parse. Expected key init_ctrs and for that
95
98
  # key to be an array of hashes.
99
+ # @param default_env [#to_h]
100
+ # Default env to merge with defined env
96
101
  # @return [Array<OodCore::Job::Adapters::Kubernetes::Resources::Container>]
97
102
  # the array of init containers
98
- def init_ctrs_from_native(ctrs)
103
+ def init_ctrs_from_native(ctrs, default_env)
99
104
  init_ctrs = []
100
105
 
101
106
  ctrs&.each do |ctr_raw|
102
- ctr = container_from_native(ctr_raw)
107
+ ctr = container_from_native(ctr_raw, default_env)
103
108
  init_ctrs.push(ctr)
104
109
  end
105
110
 
@@ -1,22 +1,45 @@
1
1
  module OodCore::Job::Adapters::Kubernetes::Resources
2
2
 
3
3
  class ConfigMap
4
- attr_accessor :name, :filename, :data
4
+ attr_accessor :name, :files
5
5
 
6
- def initialize(name, filename, data)
6
+ def initialize(name, files)
7
7
  @name = name
8
- @filename = filename
9
- @data = data
8
+ @files = []
9
+ files.each do |f|
10
+ @files << ConfigMapFile.new(f)
11
+ end
12
+ end
13
+
14
+ def mounts?
15
+ @files.any? { |f| f.mount_path }
16
+ end
17
+
18
+ def init_mounts?
19
+ @files.any? { |f| f.init_mount_path }
20
+ end
21
+ end
22
+
23
+ class ConfigMapFile
24
+ attr_accessor :filename, :data, :mount_path, :sub_path, :init_mount_path, :init_sub_path
25
+
26
+ def initialize(data)
27
+ @filename = data[:filename]
28
+ @data = data[:data]
29
+ @mount_path = data[:mount_path]
30
+ @sub_path = data[:sub_path]
31
+ @init_mount_path = data[:init_mount_path]
32
+ @init_sub_path = data[:init_sub_path]
10
33
  end
11
34
  end
12
35
 
13
36
  class Container
14
37
  attr_accessor :name, :image, :command, :port, :env, :memory, :cpu, :working_dir,
15
- :restart_policy, :supplemental_groups
38
+ :restart_policy, :image_pull_secret, :supplemental_groups
16
39
 
17
40
  def initialize(
18
- name, image, command: [], port: nil, env: [], memory: "4Gi", cpu: "1",
19
- working_dir: "", restart_policy: "Never", supplemental_groups: []
41
+ name, image, command: [], port: nil, env: {}, memory: "4Gi", cpu: "1",
42
+ working_dir: "", restart_policy: "Never", image_pull_secret: nil, supplemental_groups: []
20
43
  )
21
44
  raise ArgumentError, "containers need valid names and images" unless name && image
22
45
 
@@ -24,11 +47,12 @@ module OodCore::Job::Adapters::Kubernetes::Resources
24
47
  @image = image
25
48
  @command = command.nil? ? [] : command
26
49
  @port = port&.to_i
27
- @env = env.nil? ? [] : env
50
+ @env = env.nil? ? {} : env
28
51
  @memory = memory.nil? ? "4Gi" : memory
29
52
  @cpu = cpu.nil? ? "1" : cpu
30
53
  @working_dir = working_dir.nil? ? "" : working_dir
31
54
  @restart_policy = restart_policy.nil? ? "Never" : restart_policy
55
+ @image_pull_secret = image_pull_secret
32
56
  @supplemental_groups = supplemental_groups.nil? ? [] : supplemental_groups
33
57
  end
34
58
 
@@ -42,9 +66,9 @@ module OodCore::Job::Adapters::Kubernetes::Resources
42
66
  cpu == other.cpu &&
43
67
  working_dir == other.working_dir &&
44
68
  restart_policy == other.restart_policy &&
69
+ image_pull_secret == other.image_pull_secret &&
45
70
  supplemental_groups == other.supplemental_groups
46
71
  end
47
-
48
72
  end
49
73
 
50
74
  class PodSpec
@@ -32,6 +32,10 @@ spec:
32
32
  hostNetwork: false
33
33
  hostIPC: false
34
34
  hostPID: false
35
+ <%- unless spec.container.image_pull_secret.nil? -%>
36
+ imagePullSecrets:
37
+ - name: <%= spec.container.image_pull_secret %>
38
+ <%- end -%>
35
39
  containers:
36
40
  - name: "<%= spec.container.name %>"
37
41
  image: <%= spec.container.image %>
@@ -39,13 +43,15 @@ spec:
39
43
  <%- unless spec.container.working_dir.empty? -%>
40
44
  workingDir: "<%= spec.container.working_dir %>"
41
45
  <%- end -%>
42
- <%- unless spec.container.env.empty? -%>
43
46
  env:
44
- <%- spec.container.env.each do |env| -%>
45
- - name: <%= env[:name] %>
46
- value: "<%= env[:value] %>"
47
+ - name: POD_NAME
48
+ valueFrom:
49
+ fieldRef:
50
+ fieldPath: metadata.name
51
+ <%- spec.container.env.each_pair do |name, value| -%>
52
+ - name: <%= name %>
53
+ value: "<%= value %>"
47
54
  <%- end # for each env -%>
48
- <%- end # unless env is nil -%>
49
55
  <%- unless spec.container.command.empty? -%>
50
56
  command:
51
57
  <%- spec.container.command.each do |cmd| -%>
@@ -56,15 +62,21 @@ spec:
56
62
  ports:
57
63
  - containerPort: <%= spec.container.port %>
58
64
  <%- end -%>
65
+ <%- if configmap.mounts? || !all_mounts.empty? -%>
59
66
  volumeMounts:
60
- <%- unless configmap.nil? -%>
67
+ <%- configmap.files.each do |file| -%>
68
+ <%- next if file.mount_path.nil? -%>
61
69
  - name: configmap-volume
62
- mountPath: <%= configmap_mount_path %>
63
- <%- end -%>
70
+ mountPath: <%= file.mount_path %>
71
+ <%- unless file.sub_path.nil? -%>
72
+ subPath: <%= file.sub_path %>
73
+ <%- end # end unless file.sub_path.nil? -%>
74
+ <%- end # end configmap.files.each -%>
64
75
  <%- all_mounts.each do |mount| -%>
65
76
  - name: <%= mount[:name] %>
66
77
  mountPath: <%= mount[:destination_path] %>
67
78
  <%- end # for each mount -%>
79
+ <%- end # configmap mounts? and all_mounts not empty -%>
68
80
  resources:
69
81
  limits:
70
82
  memory: "<%= spec.container.memory %>"
@@ -83,19 +95,34 @@ spec:
83
95
  <%- spec.init_containers.each do |ctr| -%>
84
96
  - name: "<%= ctr.name %>"
85
97
  image: "<%= ctr.image %>"
98
+ env:
99
+ - name: POD_NAME
100
+ valueFrom:
101
+ fieldRef:
102
+ fieldPath: metadata.name
103
+ <%- ctr.env.each_pair do |name, value| -%>
104
+ - name: <%= name %>
105
+ value: "<%= value %>"
106
+ <%- end # for each env -%>
86
107
  command:
87
108
  <%- ctr.command.each do |cmd| -%>
88
109
  - "<%= cmd %>"
89
110
  <%- end # command loop -%>
111
+ <%- if configmap.init_mounts? || !all_mounts.empty? -%>
90
112
  volumeMounts:
91
- <%- unless configmap.nil? -%>
113
+ <%- configmap.files.each do |file| -%>
114
+ <%- next if file.init_mount_path.nil? -%>
92
115
  - name: configmap-volume
93
- mountPath: <%= configmap_mount_path %>
94
- <%- end -%>
116
+ mountPath: <%= file.init_mount_path %>
117
+ <%- unless file.init_sub_path.nil? -%>
118
+ subPath: <%= file.init_sub_path %>
119
+ <%- end # end unless file.sub_path.nil? -%>
120
+ <%- end # end configmap.files.each -%>
95
121
  <%- all_mounts.each do |mount| -%>
96
122
  - name: <%= mount[:name] %>
97
123
  mountPath: <%= mount[:destination_path] %>
98
124
  <%- end # for each mount -%>
125
+ <%- end # if config_map init mounts and all_mounts not empty -%>
99
126
  securityContext:
100
127
  allowPrivilegeEscalation: false
101
128
  capabilities:
@@ -153,6 +180,9 @@ metadata:
153
180
  labels:
154
181
  job: <%= id %>
155
182
  data:
156
- <%= configmap.filename %>: |
157
- <% config_data_lines(configmap.data).each do |line| %><%= line %><% end %>
158
- <%- end # end for configmap -%>
183
+ <%- configmap.files.each do |file| -%>
184
+ <%- next if file.data.nil? || file.filename.nil? -%>
185
+ <%= file.filename %>: |
186
+ <% config_data_lines(file.data).each do |line| %><%= line %><% end %>
187
+ <%- end # end for configmap files -%>
188
+ <%- end # end configmap.nil? %>
@@ -16,7 +16,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
16
16
  # from
17
17
  class Error < StandardError; end
18
18
 
19
- UNIT_SEPARATOR = "\x1F"
19
+ UNIT_SEPARATOR = ","
20
20
 
21
21
  # @param debug Whether the adapter should be used in debug mode
22
22
  # @param site_timeout [#to_i] A period after which the job should be killed or nil
@@ -80,12 +80,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
80
80
 
81
81
  call(*cmd, stdin: kill_cmd)
82
82
  rescue Error => e
83
- raise e unless (
84
- # The tmux server not running is not an error
85
- e.message.include?('failed to connect to server') ||
86
- # The session not being found is not an error
87
- e.message.include?("session not found: #{session_name_label}")
88
- )
83
+ interpret_and_raise(e)
89
84
  end
90
85
 
91
86
  def list_remote_sessions(host: nil)
@@ -264,8 +259,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
264
259
  |session_hash| session_hash[:session_name].start_with?(session_name_label)
265
260
  }
266
261
  rescue Error => e
267
- # The tmux server not running is not an error
268
- raise e unless e.message.include?('failed to connect to server')
262
+ interpret_and_raise(e)
269
263
  []
270
264
  end
271
265
 
@@ -280,4 +274,17 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
280
274
 
281
275
  '/dev/null'
282
276
  end
277
+
278
+ # under some conditions tmux returns status code 1 but it's not an actual
279
+ # error. These are when the session is not found or there are no sessions
280
+ # at all.
281
+ def interpret_and_raise(error)
282
+ if error.message.include?('failed to connect to server') # no sessions in tmux 1.8
283
+ nil
284
+ elsif error.message.include?('no server running on') # no sessions in tmux 2.7+ message
285
+ nil
286
+ else
287
+ raise error
288
+ end
289
+ end
283
290
  end
@@ -1,4 +1,4 @@
1
1
  module OodCore
2
2
  # The current version of {OodCore}
3
- VERSION = "0.15.1"
3
+ VERSION = "0.16.0"
4
4
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ood_core
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.15.1
4
+ version: 0.16.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Eric Franz
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: exe
12
12
  cert_chain: []
13
- date: 2021-02-25 00:00:00.000000000 Z
13
+ date: 2021-04-20 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: ood_support