ood_core 0.12.0 → 0.16.0

data/lib/ood_core/job/adapters/kubernetes/k8s_job_info.rb

@@ -0,0 +1,9 @@
+# An object that describes a submitted kubernetes job with extended information
+class OodCore::Job::Adapters::Kubernetes::K8sJobInfo < OodCore::Job::Info
+  attr_reader :ood_connection_info
+
+  def initialize(ood_connection_info: {}, **options)
+    super(options)
+    @ood_connection_info = ood_connection_info
+  end
+end

data/lib/ood_core/job/adapters/kubernetes/resources.rb

@@ -0,0 +1,82 @@
+module OodCore::Job::Adapters::Kubernetes::Resources
+
+  class ConfigMap
+    attr_accessor :name, :files
+
+    def initialize(name, files)
+      @name = name
+      @files = []
+      files.each do |f|
+        @files << ConfigMapFile.new(f)
+      end
+    end
+
+    def mounts?
+      @files.any? { |f| f.mount_path }
+    end
+
+    def init_mounts?
+      @files.any? { |f| f.init_mount_path }
+    end
+  end
+
+  class ConfigMapFile
+    attr_accessor :filename, :data, :mount_path, :sub_path, :init_mount_path, :init_sub_path
+
+    def initialize(data)
+      @filename = data[:filename]
+      @data = data[:data]
+      @mount_path = data[:mount_path]
+      @sub_path = data[:sub_path]
+      @init_mount_path = data[:init_mount_path]
+      @init_sub_path = data[:init_sub_path]
+    end
+  end
+
+  class Container
+    attr_accessor :name, :image, :command, :port, :env, :memory, :cpu, :working_dir,
+                  :restart_policy, :image_pull_secret, :supplemental_groups
+
+    def initialize(
+        name, image, command: [], port: nil, env: {}, memory: "4Gi", cpu: "1",
+        working_dir: "", restart_policy: "Never", image_pull_secret: nil, supplemental_groups: []
+      )
+      raise ArgumentError, "containers need valid names and images" unless name && image
+
+      @name = name
+      @image = image
+      @command = command.nil? ? [] : command
+      @port = port&.to_i
+      @env = env.nil? ? {} : env
+      @memory = memory.nil? ? "4Gi" : memory
+      @cpu = cpu.nil? ? "1" : cpu
+      @working_dir = working_dir.nil? ? "" : working_dir
+      @restart_policy = restart_policy.nil? ? "Never" : restart_policy
+      @image_pull_secret = image_pull_secret
+      @supplemental_groups = supplemental_groups.nil? ? [] : supplemental_groups
+    end
+
+    def ==(other)
+      name == other.name &&
+        image == other.image &&
+        command == other.command &&
+        port == other.port &&
+        env == other.env &&
+        memory == other.memory &&
+        cpu == other.cpu &&
+        working_dir == other.working_dir &&
+        restart_policy == other.restart_policy &&
+        image_pull_secret == other.image_pull_secret &&
+        supplemental_groups == other.supplemental_groups
+    end
+  end
+
+  class PodSpec
+    attr_accessor :container, :init_containers
+    def initialize(container, init_containers: nil)
+      @container = container
+      @init_containers = init_containers
+    end
+  end
+  
+end

data/lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb

@@ -0,0 +1,188 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  namespace: <%= namespace %>
+  name: <%= id %>
+  labels:
+    job: <%= id %>
+    app.kubernetes.io/name: <%= container.name %>
+    app.kubernetes.io/managed-by: open-ondemand
+    <%- if !script.accounting_id.nil? && script.accounting_id != "" -%>
+    account: <%= script.accounting_id %>
+    <%- end -%>
+  annotations:
+    <%- unless script.wall_time.nil? -%>
+    pod.kubernetes.io/lifetime: <%= helper.seconds_to_duration(script.wall_time) %>
+    <%- end -%>
+spec:
+  restartPolicy: <%= spec.container.restart_policy %>
+  securityContext:
+    runAsUser: <%= run_as_user %>
+    runAsGroup: <%= run_as_group %>
+    runAsNonRoot: true
+    <%- if spec.container.supplemental_groups.empty? -%>
+    supplementalGroups: []
+    <%- else -%>
+    supplementalGroups:
+    <%- spec.container.supplemental_groups.each do |supplemental_group| -%>
+    - "<%= supplemental_group %>"
+    <%- end -%>
+    <%- end -%>
+    fsGroup: <%= fs_group %>
+  hostNetwork: false
+  hostIPC: false
+  hostPID: false
+  <%- unless spec.container.image_pull_secret.nil? -%>
+  imagePullSecrets:
+  - name: <%= spec.container.image_pull_secret %>
+  <%- end -%>
+  containers:
+  - name: "<%= spec.container.name %>"
+    image: <%= spec.container.image %>
+    imagePullPolicy: IfNotPresent
+    <%- unless spec.container.working_dir.empty? -%>
+    workingDir: "<%= spec.container.working_dir %>"
+    <%- end -%>
+    env:
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    <%- spec.container.env.each_pair do |name, value| -%>
+    - name: <%= name %>
+      value: "<%= value %>"
+    <%- end # for each env -%>
+    <%- unless spec.container.command.empty? -%>
+    command:
+    <%- spec.container.command.each do |cmd| -%>
+    - "<%= cmd %>"
+    <%- end # for each command -%>
+    <%- end # unless command is nil -%>
+    <%- unless spec.container.port.nil? -%>
+    ports:
+    - containerPort: <%= spec.container.port %>
+    <%- end -%>
+    <%- if configmap.mounts? || !all_mounts.empty? -%>
+    volumeMounts:
+    <%- configmap.files.each do |file| -%>
+      <%- next if file.mount_path.nil? -%>
+    - name: configmap-volume
+      mountPath: <%= file.mount_path %>
+      <%- unless file.sub_path.nil? -%>
+      subPath: <%= file.sub_path %>
+      <%- end # end unless file.sub_path.nil? -%>
+    <%- end # end configmap.files.each -%>
+    <%- all_mounts.each do |mount| -%>
+    - name: <%= mount[:name] %>
+      mountPath: <%= mount[:destination_path] %>
+    <%- end # for each mount -%>
+    <%- end # configmap mounts? and all_mounts not empty -%>
+    resources:
+      limits:
+        memory: "<%= spec.container.memory %>"
+        cpu: "<%= spec.container.cpu %>"
+      requests:
+        memory: "<%= spec.container.memory %>"
+        cpu: "<%= spec.container.cpu %>"
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - all
+      privileged: false
+  <%- unless spec.init_containers.nil? -%>
+  initContainers:
+  <%- spec.init_containers.each do |ctr| -%>
+  - name: "<%= ctr.name %>"
+    image: "<%= ctr.image %>"
+    env:
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    <%- ctr.env.each_pair do |name, value| -%>
+    - name: <%= name %>
+      value: "<%= value %>"
+    <%- end # for each env -%>
+    command:
+    <%- ctr.command.each do |cmd| -%>
+    - "<%= cmd %>"
+    <%- end # command loop -%>
+    <%- if configmap.init_mounts? || !all_mounts.empty? -%>
+    volumeMounts:
+    <%- configmap.files.each do |file| -%>
+    <%- next if file.init_mount_path.nil? -%>
+    - name: configmap-volume
+      mountPath: <%= file.init_mount_path %>
+      <%- unless file.init_sub_path.nil? -%>
+      subPath: <%= file.init_sub_path %>
+      <%- end # end unless file.sub_path.nil? -%>
+    <%- end # end configmap.files.each -%>
+    <%- all_mounts.each do |mount| -%>
+    - name: <%= mount[:name] %>
+      mountPath: <%= mount[:destination_path] %>
+    <%- end # for each mount -%>
+    <%- end # if config_map init mounts and all_mounts not empty -%>
+    securityContext:
+      allowPrivilegeEscalation: false
+      capabilities:
+        drop:
+        - all
+      privileged: false
+  <%- end # init container loop -%>
+  <%- end # if init containers -%>
+  <%- unless (configmap.to_s.empty? && all_mounts.empty?) -%>
+  volumes:
+  <%- unless configmap.to_s.empty? -%>
+  - name: configmap-volume
+    configMap:
+      name: <%= configmap_name(id) %>
+  <%- end -%>
+  <%- all_mounts.each do |mount| -%>
+  <%- if mount[:type] == 'nfs' -%>
+  - name: <%= mount[:name] %>
+    nfs:
+      server: <%= mount[:host] %>
+      path: <%= mount[:path] %>
+  <%- elsif mount[:type] == 'host' -%>
+  - name: <%= mount[:name] %>
+    hostPath:
+      path: <%= mount[:path] %>
+      type: <%= mount[:host_type] %>
+  <%- end # if mount is [host,nfs] -%>
+  <%- end # for each mount -%>
+  <%- end # (configmap.to_s.empty? || all_mounts.empty?) -%>
+---
+<%- unless spec.container.port.nil? -%>
+apiVersion: v1
+kind: Service
+metadata:
+  name: <%= service_name(id) %>
+  namespace: <%= namespace %>
+  labels:
+    job: <%= id %>
+spec:
+  selector:
+    job: <%= id %>
+  ports:
+  - protocol: TCP
+    port: 80
+    targetPort: <%= spec.container.port %>
+  type: NodePort
+<%- end # end for service -%>
+---
+<%- unless configmap.nil? -%>
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: <%= configmap_name(id) %>
+  namespace: <%= namespace %>
+  labels:
+    job: <%= id %>
+data:
+<%- configmap.files.each do |file| -%>
+  <%- next if file.data.nil? || file.filename.nil? -%>
+  <%= file.filename %>: |
+    <% config_data_lines(file.data).each do |line| %><%= line %><% end %>
+<%- end # end for configmap files -%>
+<%- end # end configmap.nil? %>

data/lib/ood_core/job/adapters/linux_host/launcher.rb

@@ -16,7 +16,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
   # from
   class Error < StandardError; end
 
-  UNIT_SEPARATOR = "\x1F"
+  UNIT_SEPARATOR = ","
 
   # @param debug Whether the adapter should be used in debug mode
   # @param site_timeout [#to_i] A period after which the job should be killed or nil
@@ -80,12 +80,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
 
     call(*cmd, stdin: kill_cmd)
   rescue Error => e
-    raise e unless (
-      # The tmux server not running is not an error
-      e.message.include?('failed to connect to server') ||
-      # The session not being found is not an error
-      e.message.include?("session not found: #{session_name_label}")
-    )
+    interpret_and_raise(e)
   end
 
   def list_remote_sessions(host: nil)
@@ -166,7 +161,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
         'email_on_terminated' => script_email_on_event(script, 'terminated'),
         'email_on_start' => script_email_on_event(script, 'started'),
         'environment' => export_env(script),
-        'error_path' => (script.error_path) ? script.error_path.to_s : '/dev/null',
+        'error_path' => error_path(script),
         'job_name' => script.job_name.to_s,
         'output_path' => (script.output_path) ? script.output_path.to_s : '/dev/null',
         'script_content' => content,
@@ -176,6 +171,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
         'singularity_image' => singularity_image(script.native),
         'ssh_hosts' => ssh_hosts,
         'tmux_bin' => tmux_bin,
+        'workdir' => (script.workdir) ? script.workdir.to_s : '/tmp',
       }.each{
         |key, value| bnd.local_variable_set(key, value)
       }
@@ -263,8 +259,7 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
       |session_hash| session_hash[:session_name].start_with?(session_name_label)
     }
   rescue Error => e
-    # The tmux server not running is not an error
-    raise e unless e.message.include?('failed to connect to server')
+    interpret_and_raise(e)
     []
   end
 
@@ -272,4 +267,24 @@ class OodCore::Job::Adapters::LinuxHost::Launcher
     return false if script.content.empty?
     script.content.split("\n").first.start_with?('#!/')
   end
+
+  def error_path(script)
+    return script.error_path.to_s if script.error_path
+    return script.output_path.to_s if script.output_path
+
+    '/dev/null'
+  end
+
+  # under some conditions tmux returns status code 1 but it's not an actual
+  # error. These are when the session is not found or there are no sessions
+  # at all.
+  def interpret_and_raise(error)
+    if error.message.include?('failed to connect to server') # no sessions in tmux 1.8
+      nil
+    elsif error.message.include?('no server running on') # no sessions in tmux 2.7+ message
+      nil
+    else
+      raise error
+    end
+  end
 end

data/lib/ood_core/job/adapters/linux_host/templates/script_wrapper.erb.sh

@@ -16,13 +16,9 @@ fi
 echo $hostname
 
 # Put the script into a temp file on localhost
-<% if debug %>
-singularity_tmp_file=$(mktemp -p "$HOME" --suffix '_sing')
-tmux_tmp_file=$(mktemp -p "$HOME" --suffix "_tmux")
-<% else %>
-singularity_tmp_file=$(mktemp)
-tmux_tmp_file=$(mktemp)
-<% end %>
+singularity_tmp_file=$(mktemp -p "<%= workdir %>" --suffix '_sing')
+tmux_tmp_file=$(mktemp -p "<%= workdir %>" --suffix "_tmux")
+
 
 # Create an executable to run in a tmux session
 # The escaped HEREDOC means that we need to substitute in $singularity_tmp_file ourselves
@@ -69,10 +65,3 @@ SINGULARITY_LAUNCHER
 chmod +x "$singularity_tmp_file"
 chmod +x "$tmux_tmp_file"
 <%= tmux_bin %> new-session -d -s "<%= session_name %>" "$tmux_tmp_file"
-
-# Remove the file
-<% if ! debug %>
-# Wait 1 second to ensure that tmux session has started before the file is removed
-sleep 1
-rm -f "$tmux_tmp_file"; rm -f "$singularity_tmp_file"
-<% end %>

data/lib/ood_core/job/adapters/slurm.rb

@@ -80,6 +80,9 @@ module OodCore
           # from
           class Error < StandardError; end
 
+          # An error indicating the slurm command timed out
+          class SlurmTimeoutError < Error; end
+
           # @param cluster [#to_s, nil] the cluster name
           # @param conf [#to_s, nil] path to the slurm conf
           # @param bin [#to_s] path to slurm installation binaries
@@ -147,6 +150,9 @@ module OodCore
               end
               jobs
             end
+          rescue SlurmTimeoutError
+            # TODO: could use a log entry here
+            return [{ id: id, state: 'undetermined' }]
           end
 
           def squeue_fields(attrs)
@@ -303,7 +309,18 @@ module OodCore
 
               cmd, args = OodCore::Job::Adapters::Helper.ssh_wrap(submit_host, cmd, args, strict_host_checking)
               o, e, s = Open3.capture3(env, cmd, *(args.map(&:to_s)), stdin_data: stdin.to_s)
-              s.success? ? o : raise(Error, e)
+              s.success? ? interpret_and_raise(o, e) : raise(Error, e)
+            end
+
+            # Helper function to raise an error based on the contents of stderr.
+            # Slurm exits 0 even when the command fails, so we need to interpret stderr
+            # to see if the command was actually successful.
+            def interpret_and_raise(stdout, stderr)
+              return stdout if stderr.empty?
+
+              raise SlurmTimeoutError, stderr if /^slurm_load_jobs error: Socket timed out/.match(stderr)
+
+              stdout
             end
 
             def squeue_attrs_for_info_attrs(attrs)

data/lib/ood_core/version.rb

@@ -1,4 +1,4 @@
 module OodCore
   # The current version of {OodCore}
-  VERSION = "0.12.0"
+  VERSION = "0.16.0"
 end

data/ood_core.gemspec

@@ -24,7 +24,8 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency "ood_support", "~> 0.0.2"
   spec.add_runtime_dependency "ffi", "~> 1.9", ">= 1.9.6"
-  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "bundler", "~> 2.1"
+  spec.add_runtime_dependency "activesupport", ">= 5.2", "< 6.0"
   spec.add_development_dependency "rake", "~> 13.0.1"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "pry", "~> 0.10"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ood_core
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.16.0
 platform: ruby
 authors:
 - Eric Franz
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-08-05 00:00:00.000000000 Z
+date: 2021-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ood_support
@@ -52,14 +52,34 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '5.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '6.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -140,9 +160,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/test.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -163,8 +183,15 @@ files:
 - lib/ood_core/errors.rb
 - lib/ood_core/invalid_cluster.rb
 - lib/ood_core/job/adapter.rb
+- lib/ood_core/job/adapters/ccq.rb
 - lib/ood_core/job/adapters/drmaa.rb
 - lib/ood_core/job/adapters/helper.rb
+- lib/ood_core/job/adapters/kubernetes.rb
+- lib/ood_core/job/adapters/kubernetes/batch.rb
+- lib/ood_core/job/adapters/kubernetes/helper.rb
+- lib/ood_core/job/adapters/kubernetes/k8s_job_info.rb
+- lib/ood_core/job/adapters/kubernetes/resources.rb
+- lib/ood_core/job/adapters/kubernetes/templates/pod.yml.erb
 - lib/ood_core/job/adapters/linux_host.rb
 - lib/ood_core/job/adapters/linux_host/launcher.rb
 - lib/ood_core/job/adapters/linux_host/templates/email.erb.sh
@@ -215,7 +242,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Open OnDemand core library