ood_core 0.9.3 → 0.11.4

data/lib/ood_core/job/adapters/linux_host/launcher.rb

@@ -0,0 +1,274 @@
+require 'erb'
+require 'etc'
+require 'pathname'
+require 'securerandom'
+require 'shellwords'
+require 'time'
+
+# Object used for simplified communication SSH hosts
+#
+# @api private
+class OodCore::Job::Adapters::LinuxHost::Launcher
+  attr_reader :contain, :debug, :site_timeout, :session_name_label, :singularity_bin,
+    :site_singularity_bindpath, :default_singularity_image, :ssh_hosts,
+    :strict_host_checking, :tmux_bin, :username
+  # The root exception class that all LinuxHost adapter-specific exceptions inherit
+  # from
+  class Error < StandardError; end
+
+  UNIT_SEPARATOR = "\x1F"
+
+  # @param debug Whether the adapter should be used in debug mode
+  # @param site_timeout [#to_i] A period after which the job should be killed or nil
+  # @param singularity_bin  Path to the Singularity executable
+  # @param singularity_bindpath A comma delimited string of host paths to bindmount into the guest; sets SINGULARITY_BINDPATH environment variable
+  # @param singularity_image [#to_s] Path to the Singularity image
+  # @param ssh_hosts List of hosts to check when scanning for running jobs
+  # @param strict_host_checking Allow SSH to perform strict host checking
+  # @param submit_host The SSH-able host
+  # @param tmux_bin [#to_s] Path to the tmux executable
+  def initialize(
+    contain: false,
+    debug: false,
+    site_timeout: nil,
+    singularity_bin:,
+    singularity_bindpath: '/etc,/media,/mnt,/opt,/run,/srv,/usr,/var,/users',
+    singularity_image:,
+    ssh_hosts:,
+    strict_host_checking: false,
+    submit_host:,
+    tmux_bin:,
+    **_
+  )
+    @contain = !! contain
+    @debug = !! debug
+    @site_timeout = site_timeout.to_i
+    @session_name_label = 'launched-by-ondemand'
+    @singularity_bin = Pathname.new(singularity_bin)
+    @site_singularity_bindpath = singularity_bindpath.to_s
+    @default_singularity_image = Pathname.new(singularity_image)
+    @ssh_hosts = ssh_hosts
+    @strict_host_checking = strict_host_checking
+    @submit_host = submit_host
+    @tmux_bin = tmux_bin
+    @username = Etc.getlogin
+  end
+
+  # @param hostname [#to_s] The hostname to submit the work to
+  # @param script [OodCore::Job::Script] The script object defining the work
+  def start_remote_session(script)
+    cmd = ssh_cmd(submit_host(script), ['/usr/bin/env', 'bash'])
+
+    session_name = unique_session_name
+    output = call(*cmd, stdin: wrapped_script(script, session_name))
+    hostname = output.strip
+
+    "#{session_name}@#{hostname}"
+  end
+
+  def stop_remote_session(session_name, hostname)
+    cmd = ssh_cmd(hostname, ['/usr/bin/env', 'bash'])
+
+    kill_cmd = <<~SCRIPT
+    # Get the tmux pane PID for the target session
+    pane_pid=$(tmux list-panes -aF '\#{session_name} \#{pane_pid}' | grep '#{session_name}' | cut -f 2 -d ' ')
+    # Find the Singularity sinit PID child of the pane process
+    pane_sinit_pid=$(pstree -p -l "$pane_pid" | grep -o 'sinit([[:digit:]]*' | grep -o '[[:digit:]]*')
+    # Kill sinit which stops both Singularity-based processes and the tmux session
+    kill "$pane_sinit_pid"
+    SCRIPT
+
+    call(*cmd, stdin: kill_cmd)
+  rescue Error => e
+    raise e unless (
+      # The tmux server not running is not an error
+      e.message.include?('failed to connect to server') ||
+      # The session not being found is not an error
+      e.message.include?("session not found: #{session_name_label}")
+    )
+  end
+
+  def list_remote_sessions(host: nil)
+    host_list = (host) ? [host] : ssh_hosts
+
+    host_list.map {
+      |hostname| list_remote_tmux_session(hostname)
+    }.flatten.sort_by {
+      |hsh| hsh[:session_name]
+    }
+  end
+
+  def submit_host(script = nil)
+    if script && script.native && script.native['submit_host_override']
+      script.native['submit_host_override']
+    else
+      @submit_host
+    end
+  end
+
+  private
+
+  # Call a forked Slurm command for a given cluster
+  def call(cmd, *args, env: {}, stdin: "")
+    args  = args.map(&:to_s)
+    env = env.to_h
+    o, e, s = Open3.capture3(env, cmd, *args, stdin_data: stdin.to_s)
+    s.success? ? o : raise(Error, e)
+  end
+
+  # The full command to ssh into the destination host and execute the command.
+  # SSH options include:
+  # -t Force pseudo-terminal allocation (required to allow tmux to run)
+  # -o BatchMode=yes (set mode to be non-interactive)
+  # if ! strict_host_checking
+  # -o UserKnownHostsFile=/dev/null (do not update the user's known hosts file)
+  # -o StrictHostKeyChecking=no (do no check the user's known hosts file)
+  #
+  # @param destination_host [#to_s] the destination host you wish to ssh into
+  # @param cmd [Array<#to_s>] the command to be executed on the destination host
+  def ssh_cmd(destination_host, cmd)
+    if strict_host_checking
+      [
+        'ssh', '-t',
+        '-o', 'BatchMode=yes',
+        "#{username}@#{destination_host}"
+      ].concat(cmd)
+    else
+      [
+        'ssh', '-t',
+        '-o', 'BatchMode=yes',
+        '-o', 'UserKnownHostsFile=/dev/null',
+        '-o', 'StrictHostKeyChecking=no',
+        "#{username}@#{destination_host}"
+      ].concat(cmd)
+    end
+  end
+
+  def shell
+    ENV['SHELL'] || '/bin/bash'
+  end
+
+  # Wraps a user-provided script into a Tmux invocation
+  def wrapped_script(script, session_name)
+    content = script.content
+    unless user_script_has_shebang?(script)
+     content = "#!#{shell}\n#{content}"
+    end
+
+    ERB.new(
+      File.read(Pathname.new(__dir__).join('templates/script_wrapper.erb.sh'))
+    ).result(binding.tap {|bnd|
+      {
+        'arguments' => script_arguments(script),
+        'cd_to_workdir' => (script.workdir) ? "cd #{script.workdir}" : '',
+        'contain' => (contain) ? '--contain' : '',
+        'debug' => debug,
+        'email_on_terminated' => script_email_on_event(script, 'terminated'),
+        'email_on_start' => script_email_on_event(script, 'started'),
+        'environment' => export_env(script),
+        'error_path' => (script.error_path) ? script.error_path.to_s : '/dev/null',
+        'job_name' => script.job_name.to_s,
+        'output_path' => (script.output_path) ? script.output_path.to_s : '/dev/null',
+        'script_content' => content,
+        'script_timeout' => script_timeout(script),
+        'session_name' => session_name,
+        'singularity_bin' => singularity_bin,
+        'singularity_image' => singularity_image(script.native),
+        'tmux_bin' => tmux_bin,
+      }.each{
+        |key, value| bnd.local_variable_set(key, value)
+      }
+    })
+  end
+
+  # Generate the environment export block for this script
+  def export_env(script)
+    environment = script.job_environment
+    (environment ? environment : {}).tap{
+      |hsh|
+      hsh['SINGULARITY_BINDPATH'] = singularity_bindpath(script.native)
+    }.map{
+      |key, value| "export #{key}=#{Shellwords.escape(value)}"
+    }.sort.join("\n")
+  end
+
+  def singularity_image(native)
+    if native && native[:singularity_container]
+      return native[:singularity_container]
+    end
+
+    default_singularity_image
+  end
+
+  def singularity_bindpath(native)
+    return site_singularity_bindpath unless native && native[:singularity_bindpath]
+
+    native[:singularity_bindpath]
+  end
+
+  def script_timeout(script)
+    wall_time = script.wall_time.to_i
+    return site_timeout if wall_time == 0
+    return [wall_time, site_timeout].min unless site_timeout == 0
+
+    wall_time
+  end
+
+  def script_arguments(script)
+    return '' unless script.args
+
+    Shellwords.join(script.args)
+  end
+
+  def script_email_on_event(script, event)
+    return false unless script.email && script.send("email_on_#{event}")
+
+    ERB.new(
+      File.read(Pathname.new(__dir__).join('templates/email.erb.sh'))
+    ).result(binding.tap {|bnd|
+      {
+        'email_recipients' => script.email.map{|addr| Shellwords.escape(addr)}.join(', '),
+        'job_name' => (script.job_name) ? script.job_name : 'LinuxHost_Adapter_Job',
+        'job_status' => event
+      }.each{
+        |key, value| bnd.local_variable_set(key, value)
+      }
+    })
+  end
+
+  def unique_session_name
+    "#{session_name_label}-#{SecureRandom.uuid}"
+  end
+
+  # List all Tmux sessions on destination_host started by this adapter
+  # Additional tmux ls options available: http://man7.org/linux/man-pages/man1/tmux.1.html#FORMATS
+  def list_remote_tmux_session(destination_host)
+    # Note that the tmux variable substitution looks like Ruby string sub,
+    # these must either be single quoted strings or Ruby-string escaped as well
+    format_str = Shellwords.escape(
+      ['#{session_name}', '#{session_created}', '#{pane_pid}'].join(UNIT_SEPARATOR)
+    )
+    keys = [:session_name, :session_created, :session_pid]
+    cmd = ssh_cmd(destination_host, ['tmux', 'list-panes', '-aF', format_str])
+
+    call(*cmd).split(
+      "\n"
+    ).map do |line|
+      Hash[keys.zip(line.split(UNIT_SEPARATOR))].tap do |session_hash|
+        session_hash[:destination_host] = destination_host
+        session_hash[:id] = "#{session_hash[:session_name]}@#{destination_host}"
+      end
+    end.select{
+      |session_hash| session_hash[:session_name].start_with?(session_name_label)
+    }
+  rescue Error => e
+    # The tmux server not running is not an error
+    raise e unless e.message.include?('failed to connect to server')
+    []
+  end
+
+  def user_script_has_shebang?(script)
+    return false if script.content.empty?
+    script.content.split("\n").first.start_with?('#!/')
+  end
+end

data/lib/ood_core/job/adapters/linux_host/templates/email.erb.sh

@@ -0,0 +1,9 @@
+if command -v mail; then
+cat << EMAIL_CONTENT | mail -s "Job <%= job_name %> has <%= job_status %>" <%= email_recipients %>
+Greetings,
+
+Your job <%= job_name %> has <%= job_status %>.
+
+- The OnDemand Linux Host Adapter
+EMAIL_CONTENT
+fi

data/lib/ood_core/job/adapters/linux_host/templates/script_wrapper.erb.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+hostname
+
+# Put the script into a temp file on localhost
+<% if debug %>
+singularity_tmp_file=$(mktemp -p "$HOME" --suffix '_sing')
+tmux_tmp_file=$(mktemp -p "$HOME" --suffix "_tmux")
+<% else %>
+singularity_tmp_file=$(mktemp)
+tmux_tmp_file=$(mktemp)
+<% end %>
+
+# Create an executable to run in a tmux session
+# The escaped HEREDOC means that we need to substitute in $singularity_tmp_file ourselves
+cat << 'TMUX_LAUNCHER' | sed "s#\$singularity_tmp_file#${singularity_tmp_file}#" > "$tmux_tmp_file"
+#!/bin/bash
+<% if email_on_terminated %>
+exit_script() {
+<%# DO NOT INDENT email_on_terminated may have HEREDOCS %>
+<%= email_on_terminated %>
+trap - SIGINT SIGTERM # clear the trap
+kill -- -$$ # Sends SIGTERM to child/sub processes
+}
+trap exit_script SIGINT SIGTERM
+<% end %>
+
+<%= cd_to_workdir %>
+<%= environment %>
+
+<%= email_on_start %>
+
+# Redirect stdout and stderr to separate files for all commands run within the curly braces
+# https://unix.stackexchange.com/a/6431/204548
+# Swap sterr and stdout after stdout has been redirected
+# https://unix.stackexchange.com/a/61932/204548
+OUTPUT_PATH=<%= output_path %>
+ERROR_PATH=<%= error_path %>
+({
+timeout <%= script_timeout %>s <%= singularity_bin %> exec <%= contain %> --pid <%= singularity_image %> /bin/bash --login $singularity_tmp_file <%= arguments %>
+} | tee "$OUTPUT_PATH") 3>&1 1>&2 2>&3 | tee "$ERROR_PATH"
+
+<%= email_on_terminated %>
+
+# Exit the tmux session when we are complete
+exit 0
+TMUX_LAUNCHER
+
+# Create an executable for Singularity to run
+# Escaped HEREDOC means that we do not have to worry about Shell.escape-ing script_content
+cat << 'SINGULARITY_LAUNCHER' > "$singularity_tmp_file"
+<%= script_content %>
+SINGULARITY_LAUNCHER
+
+# Run the script inside a tmux session
+chmod +x "$singularity_tmp_file"
+chmod +x "$tmux_tmp_file"
+<%= tmux_bin %> new-session -d -s "<%= session_name %>" "$tmux_tmp_file"
+
+# Remove the file
+<% if ! debug %>
+# Wait 1 second to ensure that tmux session has started before the file is removed
+sleep 1
+rm -f "$tmux_tmp_file"; rm -f "$singularity_tmp_file"
+<% end %>

data/lib/ood_core/job/adapters/lsf.rb

@@ -167,6 +167,10 @@ module OodCore
           raise JobAdapterError, e.message
         end
 
+        def directive_prefix
+          '#BSUB'
+        end
+
         private
           # Determine state from LSF state code
           def get_state(st)

data/lib/ood_core/job/adapters/lsf/helper.rb

@@ -92,6 +92,15 @@ class OodCore::Job::Adapters::Lsf::Helper
     args += ["-W", (script.wall_time / 60).to_i] unless script.wall_time.nil?
     args += ["-L", script.shell_path.to_s] unless script.shell_path.nil?
 
+    # environment
+    env = script.job_environment || {}
+    # To preserve pre-existing behavior we only act when true or false, when nil we do nothing
+    if script.copy_environment?
+      args += ["-env", (["all"] + env.keys).join(",")]
+    elsif script.copy_environment? == false
+      args += ["-env", (["none"] + env.keys).join(",")]
+    end
+
     # input and output files
     args += ["-i", script.input_path] unless script.input_path.nil?
     args += ["-o", script.output_path] unless script.output_path.nil?
@@ -104,9 +113,6 @@ class OodCore::Job::Adapters::Lsf::Helper
 
     args += script.native unless script.native.nil?
 
-    # environment
-    env = script.job_environment || {}
-
     {args: args, env: env}
   end
 end

data/lib/ood_core/job/adapters/pbspro.rb

@@ -261,6 +261,7 @@ module OodCore
           # Set environment variables
           envvars = script.job_environment.to_h
           args += ["-v", envvars.map{|k,v| "#{k}=#{v}"}.join(",")] unless envvars.empty?
+          args += ["-V"] if script.copy_environment?
 
           # If error_path is not specified we join stdout & stderr (as this
           # mimics what the other resource managers do)
@@ -397,6 +398,10 @@ module OodCore
           raise JobAdapterError, e.message unless /Unknown Job Id/ =~ e.message || /Job has finished/ =~ e.message
         end
 
+        def directive_prefix
+          '#PBS'
+        end
+
         private
           # Convert duration to seconds
           def duration_in_seconds(time)

data/lib/ood_core/job/adapters/sge.rb

@@ -157,6 +157,10 @@ module OodCore
         rescue Batch::Error => e
           raise JobAdapterError, e.message
         end
+
+        def directive_prefix
+          '#$'
+        end
       end
     end
   end

data/lib/ood_core/job/adapters/sge/batch.rb

@@ -33,8 +33,7 @@ class OodCore::Job::Adapters::Sge::Batch
   # @see Factory.build_sge
   def initialize(config)
     @cluster          = config.fetch(:cluster, nil)
-    @conf             = Pathname.new(config.fetch(:conf, nil))
-    @bin              = Pathname.new(config.fetch(:bin, nil))
+    @bin              = Pathname.new(config.fetch(:bin, nil).to_s)
     @sge_root         = Pathname.new(config[:sge_root] || ENV['SGE_ROOT'] || "/var/lib/gridengine")
     @bin_overrides    = config.fetch(:bin_overrides, {})
 

data/lib/ood_core/job/adapters/sge/helper.rb

@@ -20,6 +20,7 @@ class OodCore::Job::Adapters::Sge::Helper
     args += ['-h'] if script.submit_as_hold
     args += ['-r', 'yes'] if script.rerunnable
     script.job_environment.each_pair {|k, v| args += ['-v', "#{k.to_s}=#{v.to_s}"]} unless script.job_environment.nil?
+    args += ["-V"] if script.copy_environment?
 
     if script.workdir
       args += ['-wd', script.workdir]

data/lib/ood_core/job/adapters/sge/qstat_xml_j_r_listener.rb

@@ -27,13 +27,15 @@ class QstatXmlJRListener
     @parsed_job = {
       :tasks => [],
       :status => :queued,
-      :procs => 1,  # un-knowable from SGE qstat output
+      :procs => 1,
       :native => {}  # TODO: improve native attribute reporting
     }
     @current_text = nil
     @current_request = nil
 
     @processing_job_array_spec = false
+    @adding_slots = false
+
     @job_array_spec = {
       start: nil,
       stop: nil,
@@ -46,6 +48,8 @@ class QstatXmlJRListener
     case name
     when 'task_id_range'
       toggle_processing_array_spec
+    when 'JB_pe_range'
+      toggle_adding_slots
     end
   end
 
@@ -78,13 +82,16 @@ class QstatXmlJRListener
     when 'djob_info'
       finalize_parsed_job
     when 'RN_min'
-      set_job_array_piece(:start)
+      set_job_array_piece(:start) if @processing_job_array_spec
+      set_slots if @adding_slots
     when 'RN_max'
-      set_job_array_piece(:stop)
+      set_job_array_piece(:stop) if @processing_job_array_spec
     when 'RN_step'
-      set_job_array_piece(:step)
+      set_job_array_piece(:step) if @processing_job_array_spec
     when 'task_id_range'
       toggle_processing_array_spec
+    when 'JB_pe_range'
+      toggle_adding_slots
     end
   end
 
@@ -186,5 +193,13 @@ class QstatXmlJRListener
   def toggle_processing_array_spec
     @processing_job_array_spec = ! @processing_job_array_spec
   end
+
+  def toggle_adding_slots
+    @adding_slots = ! @adding_slots
+  end
+
+  def set_slots
+    @parsed_job[:procs] = @current_text.to_i
+  end
 end