ood_core 0.15.1 → 0.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fde32140cb148c6ea939a1d2308446e9144aad5c853fc8c41ea839beadedf03b
-  data.tar.gz: 5925bb0f8576864a3e37696d1c5b32a258edac5ebf78d07a6d509f4ec77c2339
+  metadata.gz: 392bbf30609185792c59bc9b83bad0d2d6861885db4264a4da39356cb64624b6
+  data.tar.gz: 76df75234a646147a8fed14367d6467ffabe289bfa8264ff02614b60f6463652
 SHA512:
-  metadata.gz: c9e1d0bd9e423af5289a445c12438875ae5d74b25295c8917209f19ec69b8e84bdb74b6f4ae2da3450a344398e8d96da4ae464498a38c9146d87fff1d1bbb2dd
-  data.tar.gz: b8daebdca0ed93b8d2ebb9089657efbb2b5a88e0b78b76607090b7c5befb96fbaa4a51e820b0236dc596d76c81b8110d1d8b53090f38abbfe01e00d411c96cd5
+  metadata.gz: fd127307d7048a220bdd5bed28302e90708bb0a32c9c6905136ddb3664b8f0c5035510c528f255161c4d40c98a772d6200fa60c938aff10ea4a157fdf56d8d1f
+  data.tar.gz: f3c1ea0622da5b4387ee123843da2e0e955afd40d28e0f25e425fea010865896cdebf87171e2264ffbffbd743be8d1533971a25566251dcb8da5748191dcbeea

data/CHANGELOG.md

@@ -6,6 +6,24 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+## [0.16.0] - 2021-04-20
+### Fixed
+- tmux 2.7+ bug in the linux host adapter in [2.5.8](https://github.com/OSC/ood_core/pull/258)
+  and [259](https://github.com/OSC/ood_core/pull/259).
+
+### Changed
+
+- Changed how k8s configmaps in are defined in [251](https://github.com/OSC/ood_core/pull/251).
+  The data structure now expects a key called files which is an array of objects that hold
+  filename, data, mount_path, sub_path and init_mount_path.
+  [255](https://github.com/OSC/ood_core/pull/255) also relates to this interface change.
+
+### Added
+
+- The k8s adapter can now specify environment variables and creates defaults 
+  in [252](https://github.com/OSC/ood_core/pull/252).
+- The k8s adapter can now specify image pull secrets in [253](https://github.com/OSC/ood_core/pull/253).
+
 ## [0.15.1] - 2021-02-25
 ### Fixed
 - kubernetes adapter uses the full module for helpers in [245](https://github.com/OSC/ood_core/pull/245).
@@ -282,7 +300,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - Initial release!
 
-[Unreleased]: https://github.com/OSC/ood_core/compare/v0.15.1...HEAD
+[Unreleased]: https://github.com/OSC/ood_core/compare/v0.16.0...HEAD
+[0.15.2]: https://github.com/OSC/ood_core/compare/v0.15.1...v0.16.0
 [0.15.1]: https://github.com/OSC/ood_core/compare/v0.15.0...v0.15.1
 [0.15.0]: https://github.com/OSC/ood_core/compare/v0.14.0...v0.15.0
 [0.14.0]: https://github.com/OSC/ood_core/compare/v0.13.0...v0.14.0

data/lib/ood_core/job/adapters/kubernetes/batch.rb

@@ -112,10 +112,6 @@ class OodCore::Job::Adapters::Kubernetes::Batch
     safe_call("delete", "configmap", configmap_name(id))
   end
 
-  def configmap_mount_path
-    '/ood'
-  end
-
   private
 
   def safe_call(verb, resource, id)
@@ -153,26 +149,48 @@ class OodCore::Job::Adapters::Kubernetes::Batch
     username_prefix.nil? ? username : "#{username_prefix}-#{username}"
   end
 
+  def user
+    @user ||= Etc.getpwnam(username)
+  end
+
+  def home_dir
+    user.dir
+  end
+
   def run_as_user
-    Etc.getpwnam(username).uid
+    user.uid
   end
 
   def run_as_group
-    Etc.getpwnam(username).gid
+    user.gid
   end
 
   def fs_group
     run_as_group
   end
 
+  def group
+    Etc.getgrgid(run_as_group).name
+  end
+
+  def default_env
+    {
+      USER: username,
+      UID: run_as_user,
+      HOME: home_dir,
+      GROUP: group,
+      GID: run_as_group,
+    }
+  end
+
   # helper to template resource yml you're going to submit and
   # create an id.
   def generate_id_yml(script)
     native_data = script.native
-    container = helper.container_from_native(native_data[:container])
+    container = helper.container_from_native(native_data[:container], default_env)
     id = generate_id(container.name)
     configmap = helper.configmap_from_native(native_data, id)
-    init_containers = helper.init_ctrs_from_native(native_data[:init_containers])
+    init_containers = helper.init_ctrs_from_native(native_data[:init_containers], container.env)
     spec = OodCore::Job::Adapters::Kubernetes::Resources::PodSpec.new(container, init_containers: init_containers)
     all_mounts = native_data[:mounts].nil? ? mounts : mounts + native_data[:mounts]
 

data/lib/ood_core/job/adapters/kubernetes/helper.rb

@@ -38,18 +38,22 @@ class OodCore::Job::Adapters::Kubernetes::Helper
   #
   # @param container [#to_h]
   #   the input container hash
+  # @param default_env [#to_h]
+  #   Default env to merge with defined env
   # @return  [OodCore::Job::Adapters::Kubernetes::Resources::Container]
-  def container_from_native(container)
+  def container_from_native(container, default_env)
+    env = container.fetch(:env, {}).to_h.symbolize_keys
     OodCore::Job::Adapters::Kubernetes::Resources::Container.new(
       container[:name],
       container[:image],
       command: parse_command(container[:command]),
       port: container[:port],
-      env: container.fetch(:env, []),
+      env: default_env.merge(env),
       memory: container[:memory],
       cpu: container[:cpu],
       working_dir: container[:working_dir],
-      restart_policy: container[:restart_policy]
+      restart_policy: container[:restart_policy],
+      image_pull_secret: container[:image_pull_secret]
     )
   end
 
@@ -83,8 +87,7 @@ class OodCore::Job::Adapters::Kubernetes::Helper
 
     OodCore::Job::Adapters::Kubernetes::Resources::ConfigMap.new(
       configmap_name(id),
-      configmap[:filename],
-      configmap[:data]
+      (configmap[:files] || [])
     )
   end
 
@@ -93,13 +96,15 @@ class OodCore::Job::Adapters::Kubernetes::Helper
   # @param native_data [#to_h]
   #   the native data to parse. Expected key init_ctrs and for that
   #   key to be an array of hashes.
+  # @param default_env [#to_h]
+  #   Default env to merge with defined env
   # @return [Array<OodCore::Job::Adapters::Kubernetes::Resources::Container>]
   #   the array of init containers
-  def init_ctrs_from_native(ctrs)
+  def init_ctrs_from_native(ctrs, default_env)
     init_ctrs = []
 
     ctrs&.each do |ctr_raw|
-      ctr = container_from_native(ctr_raw)
+      ctr = container_from_native(ctr_raw, default_env)
       init_ctrs.push(ctr)
     end
 

data/lib/ood_core/job/adapters/kubernetes/resources.rb

@@ -1,22 +1,45 @@
 module OodCore::Job::Adapters::Kubernetes::Resources
 
   class ConfigMap
-    attr_accessor :name, :filename, :data
+    attr_accessor :name, :files
 
-    def initialize(name, filename, data)
+    def initialize(name, files)
       @name = name
-      @filename = filename
-      @data = data
+      @files = []
+      files.each do |f|
+        @files << ConfigMapFile.new(f)
+      end
+    end
+
+    def mounts?
+      @files.any? { |f| f.mount_path }
+    end
+
+    def init_mounts?
+      @files.any? { |f| f.init_mount_path }
+    end
+  end
+
+  class ConfigMapFile
+    attr_accessor :filename, :data, :mount_path, :sub_path, :init_mount_path, :init_sub_path
+
+    def initialize(data)
+      @filename = data[:filename]
+      @data = data[:data]
+      @mount_path = data[:mount_path]
+      @sub_path = data[:sub_path]
+      @init_mount_path = data[:init_mount_path]
+      @init_sub_path = data[:init_sub_path]
     end
   end
 
   class Container
     attr_accessor :name, :image, :command, :port, :env, :memory, :cpu, :working_dir,
-                  :restart_policy, :supplemental_groups
+                  :restart_policy, :image_pull_secret, :supplemental_groups
 
     def initialize(
-        name, image, command: [], port: nil, env: [], memory: "4Gi", cpu: "1",
-        working_dir: "", restart_policy: "Never", supplemental_groups: []
+        name, image, command: [], port: nil, env: {}, memory: "4Gi", cpu: "1",
+        working_dir: "", restart_policy: "Never", image_pull_secret: nil, supplemental_groups: []
       )
       raise ArgumentError, "containers need valid names and images" unless name && image
 
@@ -24,11 +47,12 @@ module OodCore::Job::Adapters::Kubernetes::Resources
       @image = image
       @command = command.nil? ? [] : command
       @port = port&.to_i
-      @env = env.nil? ? [] : env
+      @env = env.nil? ? {} : env
       @memory = memory.nil? ? "4Gi" : memory
       @cpu = cpu.nil? ? "1" : cpu
       @working_dir = working_dir.nil? ? "" : working_dir
       @restart_policy = restart_policy.nil? ? "Never" : restart_policy
+      @image_pull_secret = image_pull_secret
       @supplemental_groups = supplemental_groups.nil? ? [] : supplemental_groups
     end
 
@@ -42,9 +66,9 @@ module OodCore::Job::Adapters::Kubernetes::Resources
         cpu == other.cpu &&
         working_dir == other.working_dir &&
         restart_policy == other.restart_policy &&
+        image_pull_secret == other.image_pull_secret &&
         supplemental_groups == other.supplemental_groups
     end
-
   end
 
   class PodSpec

data/lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb

@@ -32,6 +32,10 @@ spec:
   hostNetwork: false
   hostIPC: false
   hostPID: false
+  <%- unless spec.container.image_pull_secret.nil? -%>
+  imagePullSecrets:
+  - name: <%= spec.container.image_pull_secret %>
+  <%- end -%>
   containers:
   - name: "<%= spec.container.name %>"
     image: <%= spec.container.image %>
@@ -39,13 +43,15 @@ spec:
     <%- unless spec.container.working_dir.empty? -%>
     workingDir: "<%= spec.container.working_dir %>"
     <%- end -%>
-    <%- unless spec.container.env.empty? -%>
     env:
-    <%- spec.container.env.each do |env| -%>
-    - name: <%= env[:name] %>
-      value: "<%= env[:value] %>"
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    <%- spec.container.env.each_pair do |name, value| -%>
+    - name: <%= name %>
+      value: "<%= value %>"
     <%- end # for each env -%>
-    <%- end # unless env is nil -%>
     <%- unless spec.container.command.empty? -%>
     command:
     <%- spec.container.command.each do |cmd| -%>
@@ -56,15 +62,21 @@ spec:
     ports:
     - containerPort: <%= spec.container.port %>
     <%- end -%>
+    <%- if configmap.mounts? || !all_mounts.empty? -%>
     volumeMounts:
-    <%- unless configmap.nil? -%>
+    <%- configmap.files.each do |file| -%>
+      <%- next if file.mount_path.nil? -%>
     - name: configmap-volume
-      mountPath:  <%= configmap_mount_path %>
-    <%- end -%>
+      mountPath: <%= file.mount_path %>
+      <%- unless file.sub_path.nil? -%>
+      subPath: <%= file.sub_path %>
+      <%- end # end unless file.sub_path.nil? -%>
+    <%- end # end configmap.files.each -%>
     <%- all_mounts.each do |mount| -%>
     - name: <%= mount[:name] %>
       mountPath: <%= mount[:destination_path] %>
     <%- end # for each mount -%>
+    <%- end # configmap mounts? and all_mounts not empty -%>
     resources:
       limits:
         memory: "<%= spec.container.memory %>"
@@ -83,19 +95,34 @@ spec:
   <%- spec.init_containers.each do |ctr| -%>
   - name: "<%= ctr.name %>"
     image: "<%= ctr.image %>"
+    env:
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    <%- ctr.env.each_pair do |name, value| -%>
+    - name: <%= name %>
+      value: "<%= value %>"
+    <%- end # for each env -%>
     command:
     <%- ctr.command.each do |cmd| -%>
     - "<%= cmd %>"
     <%- end # command loop -%>
+    <%- if configmap.init_mounts? || !all_mounts.empty? -%>
     volumeMounts:
-    <%- unless configmap.nil? -%>
+    <%- configmap.files.each do |file| -%>
+    <%- next if file.init_mount_path.nil? -%>
     - name: configmap-volume
-      mountPath:  <%= configmap_mount_path %>
-    <%- end -%>
+      mountPath: <%= file.init_mount_path %>
+      <%- unless file.init_sub_path.nil? -%>
+      subPath: <%= file.init_sub_path %>
+      <%- end # end unless file.sub_path.nil? -%>
+    <%- end # end configmap.files.each -%>
     <%- all_mounts.each do |mount| -%>
     - name: <%= mount[:name] %>
       mountPath: <%= mount[:destination_path] %>
     <%- end # for each mount -%>
+    <%- end # if config_map init mounts and all_mounts not empty -%>
     securityContext:
       allowPrivilegeEscalation: false
       capabilities:
@@ -153,6 +180,9 @@ metadata:
   labels:
     job: <%= id %>
 data:
-  <%= configmap.filename %>: |
-    <% config_data_lines(configmap.data).each do |line| %><%= line %><% end %>
-<%- end # end for configmap -%>
+<%- configmap.files.each do |file| -%>
+  <%- next if file.data.nil? || file.filename.nil? -%>
+  <%= file.filename %>: |
+    <% config_data_lines(file.data).each do |line| %><%= line %><% end %>
+<%- end # end for configmap files -%>
+<%- end # end configmap.nil? %>

data/lib/ood_core/job/adapters/linux_host/launcher.rb

@@ -16,7 +16,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
   # from
   class Error < StandardError; end
 
-  UNIT_SEPARATOR = "\x1F"
+  UNIT_SEPARATOR = ","
 
   # @param debug Whether the adapter should be used in debug mode
   # @param site_timeout [#to_i] A period after which the job should be killed or nil
@@ -80,12 +80,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
 
     call(*cmd, stdin: kill_cmd)
   rescue Error => e
-    raise e unless (
-      # The tmux server not running is not an error
-      e.message.include?('failed to connect to server') ||
-      # The session not being found is not an error
-      e.message.include?("session not found: #{session_name_label}")
-    )
+    interpret_and_raise(e)
   end
 
   def list_remote_sessions(host: nil)
@@ -264,8 +259,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
       |session_hash| session_hash[:session_name].start_with?(session_name_label)
     }
   rescue Error => e
-    # The tmux server not running is not an error
-    raise e unless e.message.include?('failed to connect to server')
+    interpret_and_raise(e)
     []
   end
 
@@ -280,4 +274,17 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
 
     '/dev/null'
   end
+
+  # under some conditions tmux returns status code 1 but it's not an actual
+  # error. These are when the session is not found or there are no sessions
+  # at all.
+  def interpret_and_raise(error)
+    if error.message.include?('failed to connect to server') # no sessions in tmux 1.8
+      nil
+    elsif error.message.include?('no server running on') # no sessions in tmux 2.7+ message
+      nil
+    else
+      raise error
+    end
+  end
 end

data/lib/ood_core/version.rb

@@ -1,4 +1,4 @@
 module OodCore
   # The current version of {OodCore}
-  VERSION = "0.15.1"
+  VERSION = "0.16.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ood_core
 version: !ruby/object:Gem::Version
-  version: 0.15.1
+  version: 0.16.0
 platform: ruby
 authors:
 - Eric Franz
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-25 00:00:00.000000000 Z
+date: 2021-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ood_support