ood_core 0.17.2 → 0.18.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: dca336fb15a08ad68f556b8d33fb76887f5c0370a0eef63685a5770fbf073110
4
- data.tar.gz: 410b08fee5e739b7444ca3054483a2758d43062af964168b3f32318489d19fa0
3
+ metadata.gz: 8564931d7e160d2b23c383d60c8001de98a1a211cd7f0418a442dd99b0c48f78
4
+ data.tar.gz: af7585fe95630e9e6aa04a004c0650ad671d36894bf1d82554bf75e87056faca
5
5
  SHA512:
6
- metadata.gz: e82540895495b9f09c92f413f8f39a894fb700122da195cd4224d68eb5eae30845f8692c6d440462bac2c4f45b0a3270e7bf5219ba4adecfc63baa3884b53d28
7
- data.tar.gz: 39a441ede8e9b91e169b1aff0c1345a56c98aac1f25a1b07d873b20e66833ed46a534b5ef23f392f5d0213b7e5c80e942f62b0031f4fd2065c327116975fdf8b
6
+ metadata.gz: bd40c9483ee3c7a3bf036c9442c7e0e5f96d819b99c63248252050ecceb7a933f8636d1e55176c3271db7503ecc65a7c29f5d0add3f4dfe85accf1a35e25949e
7
+ data.tar.gz: 3c8a28b6623b388ea75a5bc418a5d6fcb1738ee2c45f0ae78b0a54d56305ed19ccce26e9b183674633c0c67d03e93fff002438e3c750c674fe4781e948d58d90
data/CHANGELOG.md CHANGED
@@ -7,6 +7,51 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [0.18.0] - 10-18-2021
11
+
12
+ ### Fixed
13
+
14
+ - Fixed LHA crashing on strange bash output in [322](https://github.com/OSC/ood_core/pull/322).
15
+
16
+ ### Added
17
+
18
+ - All adapters now respond to #{adapter}? methods like slurm?, pbspro?, kubernetes? and so on
19
+ in [326](https://github.com/OSC/ood_core/pull/326).
20
+
21
+ ### Changed
22
+
23
+ - The kubernetes adapter now expects to set context statically in [324](https://github.com/OSC/ood_core/pull/324).
24
+ And can now accept context as a part of it's interface. It will now also always send --context when using OIDC
25
+ and that context defaults to the clustername in [327](https://github.com/OSC/ood_core/pull/327).
26
+ - Removed the activesupport dependency in [329](https://github.com/OSC/ood_core/pull/329).
27
+
28
+ ## [0.17.6] - 8-24-2021
29
+
30
+ ### Added
31
+
32
+ - kubernetes now allows for arbitrary labels to be set in [317](https://github.com/OSC/ood_core/pull/317).
33
+ - kubernetes now allows for limits and requests to be different in [318](https://github.com/OSC/ood_core/pull/318).
34
+
35
+ ## [0.17.5] - 8-20-2021
36
+
37
+ ### Fixed
38
+
39
+ - kubernetes jobs delete without waiting in [314](https://github.com/OSC/ood_core/pull/314).
40
+
41
+ ## [0.17.4] - 7-29-2021
42
+
43
+ Functionally the same as [0.17.3] but with some CI updates.
44
+
45
+ ## [0.17.3] - 7-29-2021
46
+
47
+ ### Fixed
48
+
49
+ - Fixed handling of pods in a startup phase in [303](https://github.com/OSC/ood_core/pull/303).
50
+
51
+ ### Added
52
+
53
+ - Enable automatic population of supplemental groups in [305](https://github.com/OSC/ood_core/pull/305).
54
+
10
55
  ## [0.17.2] - 7-14-2021
11
56
 
12
57
  ### Fixed
@@ -360,7 +405,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
360
405
  ### Added
361
406
  - Initial release!
362
407
 
363
- [Unreleased]: https://github.com/OSC/ood_core/compare/v0.17.2...HEAD
408
+ [Unreleased]: https://github.com/OSC/ood_core/compare/v0.18.0...HEAD
409
+ [0.18.0]: https://github.com/OSC/ood_core/compare/v0.17.8...v0.18.0
410
+ [0.17.6]: https://github.com/OSC/ood_core/compare/v0.17.5...v0.17.6
411
+ [0.17.5]: https://github.com/OSC/ood_core/compare/v0.17.4...v0.17.5
412
+ [0.17.4]: https://github.com/OSC/ood_core/compare/v0.17.3...v0.17.4
413
+ [0.17.3]: https://github.com/OSC/ood_core/compare/v0.17.2...v0.17.3
364
414
  [0.17.2]: https://github.com/OSC/ood_core/compare/v0.17.1...v0.17.2
365
415
  [0.17.1]: https://github.com/OSC/ood_core/compare/v0.17.0...v0.17.1
366
416
  [0.17.0]: https://github.com/OSC/ood_core/compare/v0.16.1...v0.17.0
data/RAILS-LICENSE ADDED
@@ -0,0 +1,20 @@
1
+ Copyright (c) 2005-2021 David Heinemeier Hansson
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining
4
+ a copy of this software and associated documentation files (the
5
+ "Software"), to deal in the Software without restriction, including
6
+ without limitation the rights to use, copy, modify, merge, publish,
7
+ distribute, sublicense, and/or sell copies of the Software, and to
8
+ permit persons to whom the Software is furnished to do so, subject to
9
+ the following conditions:
10
+
11
+ The above copyright notice and this permission notice shall be
12
+ included in all copies or substantial portions of the Software.
13
+
14
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -63,6 +63,20 @@ module OodCore
63
63
  @errors = c.fetch(:errors, []) .to_a
64
64
  end
65
65
 
66
+
67
+ # programatically define some methods like slurm? or torque?
68
+ Dir.entries("#{__dir__}/job/adapters").select do |f|
69
+ File.file?("#{__dir__}/job/adapters/#{f}") && File.extname(f) == '.rb'
70
+ end.map do |f|
71
+ File.basename(f, '.rb')
72
+ end.reject do |f|
73
+ ['helper', 'drmaa' ].include?(f)
74
+ end.each do |adapter|
75
+ define_method(:"#{adapter}?") do
76
+ job_config.fetch(:adapter, nil) == adapter
77
+ end
78
+ end
79
+
66
80
  # Metadata that provides extra information about this cluster
67
81
  # @return [OpenStruct] the metadata
68
82
  def metadata
@@ -11,30 +11,27 @@ class OodCore::Job::Adapters::Kubernetes::Batch
11
11
  class Error < StandardError; end
12
12
  class NotFoundError < StandardError; end
13
13
 
14
- attr_reader :config_file, :bin, :cluster, :mounts
15
- attr_reader :all_namespaces, :using_context, :helper
14
+ attr_reader :config_file, :bin, :cluster, :context, :mounts
15
+ attr_reader :all_namespaces, :helper
16
16
  attr_reader :username_prefix, :namespace_prefix
17
+ attr_reader :auto_supplemental_groups
17
18
 
18
19
  def initialize(options = {})
19
20
  options = options.to_h.symbolize_keys
20
21
 
21
- @config_file = options.fetch(:config_file, default_config_file)
22
+ @config_file = options.fetch(:config_file, Batch.default_config_file)
22
23
  @bin = options.fetch(:bin, '/usr/bin/kubectl')
23
24
  @cluster = options.fetch(:cluster, 'open-ondemand')
24
25
  @mounts = options.fetch(:mounts, []).map { |m| m.to_h.symbolize_keys }
25
26
  @all_namespaces = options.fetch(:all_namespaces, false)
26
27
  @username_prefix = options.fetch(:username_prefix, '')
27
28
  @namespace_prefix = options.fetch(:namespace_prefix, '')
29
+ @auto_supplemental_groups = options.fetch(:auto_supplemental_groups, false)
28
30
 
29
- @using_context = false
30
- @helper = OodCore::Job::Adapters::Kubernetes::Helper.new
31
+ tmp_ctx = options.fetch(:context, nil)
32
+ @context = tmp_ctx.nil? && oidc_auth?(options.fetch(:auth, {})) ? @cluster : tmp_ctx
31
33
 
32
- begin
33
- make_kubectl_config(options)
34
- rescue
35
- # FIXME could use a log here
36
- # means you couldn't 'kubectl set config'
37
- end
34
+ @helper = OodCore::Job::Adapters::Kubernetes::Helper.new
38
35
  end
39
36
 
40
37
  def resource_file(resource_type = 'pod')
@@ -59,10 +56,10 @@ class OodCore::Job::Adapters::Kubernetes::Batch
59
56
  end
60
57
 
61
58
  def info_all(attrs: nil)
62
- cmd = if all_namespaces
63
- "#{base_cmd} get pods -o json --all-namespaces"
59
+ cmd = if @all_namespaces
60
+ "#{base_cmd} -o json get pods --all-namespaces"
64
61
  else
65
- "#{namespaced_cmd} get pods -o json"
62
+ "#{namespaced_cmd} -o json get pods"
66
63
  end
67
64
 
68
65
  output = call(cmd)
@@ -115,6 +112,32 @@ class OodCore::Job::Adapters::Kubernetes::Batch
115
112
  safe_call("delete", "configmap", configmap_name(id))
116
113
  end
117
114
 
115
+ class << self
116
+ def default_config_file
117
+ (ENV['KUBECONFIG'] || "#{Dir.home}/.kube/config")
118
+ end
119
+
120
+ def default_auth
121
+ {
122
+ type: 'managed'
123
+ }.symbolize_keys
124
+ end
125
+
126
+ def default_server
127
+ {
128
+ endpoint: 'https://localhost:8080',
129
+ cert_authority_file: nil
130
+ }.symbolize_keys
131
+ end
132
+
133
+ def configure_kube!(config)
134
+ k = self.new(config)
135
+ # TODO: probably shouldn't be using send here
136
+ k.send(:set_cluster, config.fetch(:server, default_server).to_h.symbolize_keys)
137
+ k.send(:configure_auth, config.fetch(:auth, default_auth).to_h.symbolize_keys)
138
+ end
139
+ end
140
+
118
141
  private
119
142
 
120
143
  def safe_call(verb, resource, id)
@@ -123,7 +146,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
123
146
  when "get"
124
147
  call_json_output('get', resource, id)
125
148
  when "delete"
126
- call("#{namespaced_cmd} delete #{resource} #{id}")
149
+ call("#{namespaced_cmd} delete #{resource} #{id} --wait=false")
127
150
  end
128
151
  rescue NotFoundError
129
152
  {}
@@ -176,6 +199,19 @@ class OodCore::Job::Adapters::Kubernetes::Batch
176
199
  Etc.getgrgid(run_as_group).name
177
200
  end
178
201
 
202
+ def default_supplemental_groups
203
+ OodSupport::User.new.groups.sort_by(&:id).map(&:id).reject { |id| id < 1000 }
204
+ end
205
+
206
+ def supplemental_groups(groups = [])
207
+ sgroups = []
208
+ if auto_supplemental_groups
209
+ sgroups.concat(default_supplemental_groups)
210
+ end
211
+ sgroups.concat(groups.to_a)
212
+ sgroups.uniq.sort
213
+ end
214
+
179
215
  def default_env
180
216
  {
181
217
  USER: username,
@@ -191,6 +227,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
191
227
  # create an id.
192
228
  def generate_id_yml(script)
193
229
  native_data = script.native
230
+ native_data[:container][:supplemental_groups] = supplemental_groups(native_data[:container][:supplemental_groups])
194
231
  container = helper.container_from_native(native_data[:container], default_env)
195
232
  id = generate_id(container.name)
196
233
  configmap = helper.configmap_from_native(native_data, id, script.content)
@@ -211,6 +248,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
211
248
  # and id=my-pod-id
212
249
  def call_json_output(verb, resource, id, stdin: nil)
213
250
  cmd = "#{formatted_ns_cmd} #{verb} #{resource} #{id}"
251
+
214
252
  data = call(cmd, stdin: stdin)
215
253
  data = data.empty? ? '{}' : data
216
254
  json_data = JSON.parse(data, symbolize_names: true)
@@ -234,27 +272,6 @@ class OodCore::Job::Adapters::Kubernetes::Batch
234
272
  "#{namespace_prefix}#{username}"
235
273
  end
236
274
 
237
- def context
238
- cluster
239
- end
240
-
241
- def default_config_file
242
- (ENV['KUBECONFIG'] || "#{Dir.home}/.kube/config")
243
- end
244
-
245
- def default_auth
246
- {
247
- type: 'managaged'
248
- }.symbolize_keys
249
- end
250
-
251
- def default_server
252
- {
253
- endpoint: 'https://localhost:8080',
254
- cert_authority_file: nil
255
- }.symbolize_keys
256
- end
257
-
258
275
  def formatted_ns_cmd
259
276
  "#{namespaced_cmd} -o json"
260
277
  end
@@ -265,7 +282,7 @@ class OodCore::Job::Adapters::Kubernetes::Batch
265
282
 
266
283
  def base_cmd
267
284
  base = "#{bin} --kubeconfig=#{config_file}"
268
- base << " --context=#{context}" if using_context
285
+ base << " --context=#{context}" if context?
269
286
  base
270
287
  end
271
288
 
@@ -293,28 +310,30 @@ class OodCore::Job::Adapters::Kubernetes::Batch
293
310
  nil
294
311
  end
295
312
 
296
- def make_kubectl_config(config)
297
- set_cluster(config.fetch(:server, default_server).to_h.symbolize_keys)
298
- configure_auth(config.fetch(:auth, default_auth).to_h.symbolize_keys)
299
- end
300
-
301
313
  def configure_auth(auth)
302
- type = auth.fetch(:type)
303
- return if managed?(type)
304
-
305
- case type
306
- when 'gke'
314
+ if managed_auth?(auth)
315
+ return
316
+ elsif gke_auth?(auth)
307
317
  set_gke_config(auth)
308
- when 'oidc'
318
+ elsif oidc_auth?(auth)
309
319
  set_context
310
320
  end
311
321
  end
312
322
 
313
- def use_context
314
- @using_context = true
323
+ def context?
324
+ !@context.nil?
325
+ end
326
+
327
+ def gke_auth?(auth = {})
328
+ auth.fetch(:type, nil) == 'gke'
329
+ end
330
+
331
+ def oidc_auth?(auth = {})
332
+ auth.fetch(:type, nil) == 'oidc'
315
333
  end
316
334
 
317
- def managed?(type)
335
+ def managed_auth?(auth = {})
336
+ type = auth.fetch(:type, nil)
318
337
  if type.nil?
319
338
  true # maybe should be false?
320
339
  else
@@ -343,23 +362,24 @@ class OodCore::Job::Adapters::Kubernetes::Batch
343
362
  # gke cluster name can probably can differ from what ood calls the cluster
344
363
  cmd = "gcloud container clusters get-credentials #{locale} #{cluster}"
345
364
  env = { 'KUBECONFIG' => config_file }
346
- call(cmd, env)
365
+ call(cmd, env: env)
347
366
  end
348
367
 
349
368
  def set_context
350
- cmd = "#{base_cmd} config set-context #{cluster}"
369
+ # can't really use base_cmd, bc it may use --context flag
370
+ cmd = "#{bin} --kubeconfig=#{config_file} config set-context #{context}"
351
371
  cmd << " --cluster=#{cluster} --namespace=#{namespace}"
352
372
  cmd << " --user=#{k8s_username}"
353
373
 
354
374
  call(cmd)
355
- use_context
356
375
  end
357
376
 
358
377
  def set_cluster(config)
359
378
  server = config.fetch(:endpoint)
360
379
  cert = config.fetch(:cert_authority_file, nil)
361
380
 
362
- cmd = "#{base_cmd} config set-cluster #{cluster}"
381
+ # shouldn't use context here either
382
+ cmd = "#{bin} --kubeconfig=#{config_file} config set-cluster #{cluster}"
363
383
  cmd << " --server=#{server}"
364
384
  cmd << " --certificate-authority=#{cert}" unless cert.nil?
365
385
 
@@ -4,7 +4,9 @@ class OodCore::Job::Adapters::Kubernetes::Helper
4
4
  require_relative 'k8s_job_info'
5
5
  require 'resolv'
6
6
  require 'base64'
7
- require 'active_support/core_ext/hash'
7
+ require 'ood_core/refinements/hash_extensions'
8
+
9
+ using OodCore::Refinements::HashExtensions
8
10
 
9
11
  class K8sDataError < StandardError; end
10
12
 
@@ -49,13 +51,17 @@ class OodCore::Job::Adapters::Kubernetes::Helper
49
51
  command: parse_command(container[:command]),
50
52
  port: container[:port],
51
53
  env: default_env.merge(env),
52
- memory: container[:memory],
53
- cpu: container[:cpu],
54
+ memory_limit: container[:memory_limit] || container[:memory],
55
+ memory_request: container[:memory_request] || container[:memory],
56
+ cpu_limit: container[:cpu_limit] || container[:cpu],
57
+ cpu_request: container[:cpu_request] || container[:cpu],
54
58
  working_dir: container[:working_dir],
55
59
  restart_policy: container[:restart_policy],
56
60
  image_pull_policy: container[:image_pull_policy],
57
61
  image_pull_secret: container[:image_pull_secret],
62
+ supplemental_groups: container[:supplemental_groups],
58
63
  startup_probe: container[:startup_probe],
64
+ labels: container[:labels],
59
65
  )
60
66
  end
61
67
 
@@ -266,7 +272,9 @@ class OodCore::Job::Adapters::Kubernetes::Helper
266
272
  container_statuses = json_data.dig(:status, :containerStatuses)
267
273
  unschedulable = conditions.to_a.any? { |c| c.dig(:reason) == "Unschedulable" }
268
274
  ready = !container_statuses.to_a.empty? && container_statuses.to_a.all? { |s| s.dig(:ready) == true }
275
+ started = !container_statuses.to_a.empty? && container_statuses.to_a.any? { |s| s.fetch(:state, {}).key?(:running) }
269
276
  return "running" if ready
277
+ return "queued" if phase == "Running" && started
270
278
 
271
279
  state = case phase
272
280
  when "Pending"
@@ -55,14 +55,16 @@ module OodCore::Job::Adapters::Kubernetes::Resources
55
55
  end
56
56
 
57
57
  class Container
58
- attr_accessor :name, :image, :command, :port, :env, :memory, :cpu, :working_dir,
58
+ attr_accessor :name, :image, :command, :port, :env, :working_dir,
59
+ :memory_limit, :memory_request, :cpu_limit, :cpu_request,
59
60
  :restart_policy, :image_pull_policy, :image_pull_secret, :supplemental_groups,
60
- :startup_probe
61
+ :startup_probe, :labels
61
62
 
62
63
  def initialize(
63
- name, image, command: [], port: nil, env: {}, memory: "4Gi", cpu: "1",
64
+ name, image, command: [], port: nil, env: {},
65
+ memory_limit: nil, memory_request: nil, cpu_limit: nil, cpu_request: nil,
64
66
  working_dir: "", restart_policy: "Never", image_pull_policy: nil, image_pull_secret: nil, supplemental_groups: [],
65
- startup_probe: {}
67
+ startup_probe: {}, labels: {}
66
68
  )
67
69
  raise ArgumentError, "containers need valid names and images" unless name && image
68
70
 
@@ -71,14 +73,17 @@ module OodCore::Job::Adapters::Kubernetes::Resources
71
73
  @command = command.nil? ? [] : command
72
74
  @port = port&.to_i
73
75
  @env = env.nil? ? {} : env
74
- @memory = memory.nil? ? "4Gi" : memory
75
- @cpu = cpu.nil? ? "1" : cpu
76
+ @memory_limit = memory_limit.nil? ? "4Gi" : memory_limit
77
+ @memory_request = memory_request.nil? ? "4Gi" : memory_request
78
+ @cpu_limit = cpu_limit.nil? ? "1" : cpu_limit
79
+ @cpu_request = cpu_request.nil? ? "1" : cpu_request
76
80
  @working_dir = working_dir.nil? ? "" : working_dir
77
81
  @restart_policy = restart_policy.nil? ? "Never" : restart_policy
78
82
  @image_pull_policy = image_pull_policy.nil? ? "IfNotPresent" : image_pull_policy
79
83
  @image_pull_secret = image_pull_secret
80
84
  @supplemental_groups = supplemental_groups.nil? ? [] : supplemental_groups
81
85
  @startup_probe = TCPProbe.new(@port, startup_probe)
86
+ @labels = labels.nil? ? {} : labels
82
87
  end
83
88
 
84
89
  def ==(other)
@@ -87,14 +92,17 @@ module OodCore::Job::Adapters::Kubernetes::Resources
87
92
  command == other.command &&
88
93
  port == other.port &&
89
94
  env == other.env &&
90
- memory == other.memory &&
91
- cpu == other.cpu &&
95
+ memory_limit == other.memory_limit &&
96
+ memory_request == other.memory_request &&
97
+ cpu_limit == other.cpu_limit &&
98
+ cpu_request == other.cpu_request &&
92
99
  working_dir == other.working_dir &&
93
100
  restart_policy == other.restart_policy &&
94
101
  image_pull_policy == other.image_pull_policy &&
95
102
  image_pull_secret == other.image_pull_secret &&
96
103
  supplemental_groups == other.supplemental_groups &&
97
- startup_probe.to_h == other.startup_probe.to_h
104
+ startup_probe.to_h == other.startup_probe.to_h &&
105
+ labels.to_h == other.labels.to_h
98
106
  end
99
107
  end
100
108
 
@@ -10,6 +10,9 @@ metadata:
10
10
  <%- if !script.accounting_id.nil? && script.accounting_id != "" -%>
11
11
  account: <%= script.accounting_id %>
12
12
  <%- end -%>
13
+ <%- spec.container.labels.each_pair do |key, value| -%>
14
+ <%= key %>: "<%= value %>"
15
+ <%- end -%>
13
16
  annotations:
14
17
  <%- unless script.wall_time.nil? -%>
15
18
  pod.kubernetes.io/lifetime: <%= helper.seconds_to_duration(script.wall_time) %>
@@ -20,14 +23,7 @@ spec:
20
23
  runAsUser: <%= run_as_user %>
21
24
  runAsGroup: <%= run_as_group %>
22
25
  runAsNonRoot: true
23
- <%- if spec.container.supplemental_groups.empty? -%>
24
- supplementalGroups: []
25
- <%- else -%>
26
- supplementalGroups:
27
- <%- spec.container.supplemental_groups.each do |supplemental_group| -%>
28
- - "<%= supplemental_group %>"
29
- <%- end -%>
30
- <%- end -%>
26
+ supplementalGroups: <%= spec.container.supplemental_groups %>
31
27
  fsGroup: <%= fs_group %>
32
28
  hostNetwork: false
33
29
  hostIPC: false
@@ -95,14 +91,14 @@ spec:
95
91
  <%- end # configmap mounts? and all_mounts not empty -%>
96
92
  resources:
97
93
  limits:
98
- memory: "<%= spec.container.memory %>"
99
- cpu: "<%= spec.container.cpu %>"
94
+ memory: "<%= spec.container.memory_limit %>"
95
+ cpu: "<%= spec.container.cpu_limit %>"
100
96
  <%- unless script.gpus_per_node.nil? -%>
101
97
  <%= gpu_type %>: <%= script.gpus_per_node %>
102
98
  <%- end -%>
103
99
  requests:
104
- memory: "<%= spec.container.memory %>"
105
- cpu: "<%= spec.container.cpu %>"
100
+ memory: "<%= spec.container.memory_request %>"
101
+ cpu: "<%= spec.container.cpu_request %>"
106
102
  <%- unless script.gpus_per_node.nil? -%>
107
103
  <%= gpu_type %>: <%= script.gpus_per_node %>
108
104
  <%- end -%>
@@ -61,7 +61,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
61
61
 
62
62
  session_name = unique_session_name
63
63
  output = call(*cmd, stdin: wrapped_script(script, session_name))
64
- hostname = output.strip
64
+ hostname = parse_hostname(output)
65
65
 
66
66
  "#{session_name}@#{hostname}"
67
67
  end
@@ -242,22 +242,20 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
242
242
  def list_remote_tmux_session(destination_host)
243
243
  # Note that the tmux variable substitution looks like Ruby string sub,
244
244
  # these must either be single quoted strings or Ruby-string escaped as well
245
- format_str = Shellwords.escape(
246
- ['#{session_name}', '#{session_created}', '#{pane_pid}'].join(UNIT_SEPARATOR)
247
- )
245
+ format_str = Shellwords.escape(['#{session_name}', '#{session_created}', '#{pane_pid}'].join(UNIT_SEPARATOR))
248
246
  keys = [:session_name, :session_created, :session_pid]
249
247
  cmd = ssh_cmd(destination_host, ['tmux', 'list-panes', '-aF', format_str])
250
-
251
- call(*cmd).split(
252
- "\n"
253
- ).map do |line|
248
+
249
+ call(*cmd).split("\n").map do |line|
254
250
  Hash[keys.zip(line.split(UNIT_SEPARATOR))].tap do |session_hash|
255
251
  session_hash[:destination_host] = destination_host
256
252
  session_hash[:id] = "#{session_hash[:session_name]}@#{destination_host}"
257
253
  end
258
- end.select{
259
- |session_hash| session_hash[:session_name].start_with?(session_name_label)
260
- }
254
+ end.select do |session_hash|
255
+ session_hash.compact.length >= 5 &&
256
+ !session_hash[:session_name].nil? &&
257
+ session_hash[:session_name].start_with?(session_name_label)
258
+ end
261
259
  rescue Error => e
262
260
  interpret_and_raise(e)
263
261
  []
@@ -287,4 +285,10 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
287
285
  raise error
288
286
  end
289
287
  end
288
+
289
+ def parse_hostname(output)
290
+ output.split($/).map do |line|
291
+ line.match(/^(([\w+]|[a-zA-Z0-9][\w*-]*\.))*$/)
292
+ end.compact.last.to_s
293
+ end
290
294
  end
@@ -13,6 +13,7 @@ if [ -z "$hostname" ]; then
13
13
  exit 1
14
14
  fi
15
15
 
16
+ echo ""
16
17
  echo $hostname
17
18
 
18
19
  # Put the script into a temp file on localhost
@@ -2,6 +2,8 @@ module OodCore
2
2
  # Namespace for Ruby refinements
3
3
  module Refinements
4
4
  # This module provides refinements for manipulating the Ruby {Hash} class.
5
+ # Some elements have been taken from Rails (https://github.com/rails/rails)
6
+ # and it's LICENSE has been added as RAILS-LICENSE in the root directory of this project.
5
7
  module HashExtensions
6
8
  refine Hash do
7
9
  # Symbolize the keys in a {Hash}
@@ -28,6 +30,37 @@ module OodCore
28
30
  def compact
29
31
  self.select { |_, value| !value.nil? }
30
32
  end
33
+
34
+ # Returns a new hash with +self+ and +other_hash+ merged recursively.
35
+ #
36
+ # h1 = { a: true, b: { c: [1, 2, 3] } }
37
+ # h2 = { a: false, b: { x: [3, 4, 5] } }
38
+ #
39
+ # h1.deep_merge(h2) # => { a: false, b: { c: [1, 2, 3], x: [3, 4, 5] } }
40
+ #
41
+ # Like with Hash#merge in the standard library, a block can be provided
42
+ # to merge values:
43
+ #
44
+ # h1 = { a: 100, b: 200, c: { c1: 100 } }
45
+ # h2 = { b: 250, c: { c1: 200 } }
46
+ # h1.deep_merge(h2) { |key, this_val, other_val| this_val + other_val }
47
+ # # => { a: 100, b: 450, c: { c1: 300 } }
48
+ def deep_merge(other_hash, &block)
49
+ dup.deep_merge!(other_hash, &block)
50
+ end
51
+
52
+ # Same as +deep_merge+, but modifies +self+.
53
+ def deep_merge!(other_hash, &block)
54
+ merge!(other_hash) do |key, this_val, other_val|
55
+ if this_val.is_a?(Hash) && other_val.is_a?(Hash)
56
+ this_val.deep_merge(other_val, &block)
57
+ elsif block_given?
58
+ block.call(key, this_val, other_val)
59
+ else
60
+ other_val
61
+ end
62
+ end
63
+ end
31
64
  end
32
65
  end
33
66
  end
@@ -1,4 +1,4 @@
1
1
  module OodCore
2
2
  # The current version of {OodCore}
3
- VERSION = "0.17.2"
3
+ VERSION = "0.18.0"
4
4
  end
data/ood_core.gemspec CHANGED
@@ -15,7 +15,7 @@ Gem::Specification.new do |spec|
15
15
  spec.license = "MIT"
16
16
 
17
17
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
18
- f.match(%r{^(test|spec|features)/})
18
+ f.match(%r{^(test|spec|features|.github)/})
19
19
  end
20
20
  spec.bindir = "exe"
21
21
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -25,7 +25,6 @@ Gem::Specification.new do |spec|
25
25
  spec.add_runtime_dependency "ood_support", "~> 0.0.2"
26
26
  spec.add_runtime_dependency "ffi", "~> 1.9", ">= 1.9.6"
27
27
  spec.add_development_dependency "bundler", "~> 2.1"
28
- spec.add_runtime_dependency "activesupport", ">= 5.2", "< 6.0"
29
28
  spec.add_development_dependency "rake", "~> 13.0.1"
30
29
  spec.add_development_dependency "rspec", "~> 3.0"
31
30
  spec.add_development_dependency "pry", "~> 0.10"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ood_core
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.17.2
4
+ version: 0.18.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Eric Franz
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: exe
12
12
  cert_chain: []
13
- date: 2021-07-16 00:00:00.000000000 Z
13
+ date: 2021-10-18 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: ood_support
@@ -60,26 +60,6 @@ dependencies:
60
60
  - - "~>"
61
61
  - !ruby/object:Gem::Version
62
62
  version: '2.1'
63
- - !ruby/object:Gem::Dependency
64
- name: activesupport
65
- requirement: !ruby/object:Gem::Requirement
66
- requirements:
67
- - - ">="
68
- - !ruby/object:Gem::Version
69
- version: '5.2'
70
- - - "<"
71
- - !ruby/object:Gem::Version
72
- version: '6.0'
73
- type: :runtime
74
- prerelease: false
75
- version_requirements: !ruby/object:Gem::Requirement
76
- requirements:
77
- - - ">="
78
- - !ruby/object:Gem::Version
79
- version: '5.2'
80
- - - "<"
81
- - !ruby/object:Gem::Version
82
- version: '6.0'
83
63
  - !ruby/object:Gem::Dependency
84
64
  name: rake
85
65
  requirement: !ruby/object:Gem::Requirement
@@ -160,13 +140,12 @@ executables: []
160
140
  extensions: []
161
141
  extra_rdoc_files: []
162
142
  files:
163
- - ".github/dependabot.yml"
164
- - ".github/workflows/test.yml"
165
143
  - ".gitignore"
166
144
  - ".rspec"
167
145
  - CHANGELOG.md
168
146
  - Gemfile
169
147
  - LICENSE.txt
148
+ - RAILS-LICENSE
170
149
  - README.md
171
150
  - Rakefile
172
151
  - bin/console
@@ -1,8 +0,0 @@
1
- version: 2
2
- updates:
3
- - package-ecosystem: bundler
4
- directory: "/"
5
- schedule:
6
- interval: daily
7
- time: "03:30"
8
- open-pull-requests-limit: 10
@@ -1,30 +0,0 @@
1
- name: Unit Tests
2
-
3
- on:
4
- push:
5
- branches:
6
- - master
7
- pull_request:
8
- branches:
9
- - master
10
-
11
- jobs:
12
- tests:
13
- runs-on: ubuntu-latest
14
-
15
- steps:
16
- - name: checkout
17
- uses: actions/checkout@v2
18
-
19
- - name: Setup Ruby using Bundler
20
- uses: ruby/setup-ruby@v1
21
- with:
22
- ruby-version: "2.7.1"
23
- bundler-cache: true
24
- bundler: "2.1.4"
25
-
26
- - name: install gems
27
- run: bundle install
28
-
29
- - name: test
30
- run: bundle exec rake spec