ood_core 0.9.3 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 40ab5af1e1955af503e10a47b233d7b846ed5f16
-  data.tar.gz: 85c9e94a1d4a16b1d444ac02230fba8a8a7f5df9
+  metadata.gz: c668d456d8773bfa4a3af714f73a57d2f9396ee8
+  data.tar.gz: 14cdd12014850a9464505d7e34e5e4397826c6d9
 SHA512:
-  metadata.gz: 4d617a6ab11b960e3a74d773084a7f8125601e1e27f7cca1b3ffe658abded249718481905282100a569f138eab07310171469c9135c7d82cbc91c9f7d2b99a85
-  data.tar.gz: 03f198b479fdbd06c1fe14d224db3400fce531c6d68abe7c3bdfc0fef4684d550aea054d713860a6831e92b46e3e35600193c6b186f5aa7681740618042e502a
+  metadata.gz: ebb2ea211dc272f03884faf0c2245e2f2b945851023ef5f5f26c1862514a0626a57c35c3be636d9287d1581d56dd18a3e6ff014e41837d63b7d05559605acd79
+  data.tar.gz: beb0f071c17c7632e9aae32ee243e3fd47480ff385d1d82cfe42e48652a6bbaaee3db7c224c7c2b4c8f91b11b11b8e5d9170b231f846e60ff9e6e1d2f03a0260

data/CHANGELOG.md

@@ -6,6 +6,13 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+## [0.10.0] - 2019-11-05
+### Added
+- Added an adapter for submitting work on Linux hosted systems without using a scheduler
+
+### Fixed
+- Fixed bug where an unreadable cluster config would cause crashes
+
 ## [0.9.3] - 2019-05-08
 ### Fixed
 - Fixed bug relating to cluster comparison
@@ -27,7 +34,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - Grid Engine adapter now starts scripts in the current directory like all other adapters
 - Fixed issue where Slurm comment field might break job info parsing
 - Fixed possible crash when comparing two clusters if the id of one of the clusters is nil
-- Fixed bug with the live system test that impacted non-Torque systems
+- Fixed bug with the live system test that impacted non-LSF systems
 - Fixed bug with Slurm adapter when submit time is not available
 
 ## [0.8.0] - 2019-01-29
@@ -189,7 +196,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - Initial release!
 
-[Unreleased]: https://github.com/OSC/ood_core/compare/v0.9.3...HEAD
+[Unreleased]: https://github.com/OSC/ood_core/compare/v0.10.0...HEAD
+[0.10.0]: https://github.com/OSC/ood_core/compare/v0.9.3...v0.10.0
 [0.9.3]: https://github.com/OSC/ood_core/compare/v0.9.2...v0.9.3
 [0.9.2]: https://github.com/OSC/ood_core/compare/v0.9.1...v0.9.2
 [0.9.1]: https://github.com/OSC/ood_core/compare/v0.9.0...v0.9.1

data/lib/ood_core/clusters.rb

@@ -19,20 +19,18 @@ module OodCore
 
         clusters = []
         if config.file?
-          CONFIG_VERSION.any? do |version|
-            YAML.safe_load(config.read).fetch(version, {}).each do |k, v|
-              clusters << Cluster.new(send("parse_#{version}", id: k, cluster: v))
+          if config.readable?
+            CONFIG_VERSION.any? do |version|
+              YAML.safe_load(config.read).fetch(version, {}).each do |k, v|
+                clusters << Cluster.new(send("parse_#{version}", id: k, cluster: v))
+              end
             end
-            !clusters.empty?
           end
         elsif config.directory?
-          Pathname.glob(config.join("*.yml")).each do |p|
+          Pathname.glob(config.join("*.yml")).select(&:file?).select(&:readable?).each do |p|
             CONFIG_VERSION.any? do |version|
               if cluster = YAML.safe_load(p.read).fetch(version, nil)
                 clusters << Cluster.new(send("parse_#{version}", id: p.basename(".yml").to_s, cluster: cluster))
-                true
-              else
-                false
               end
             end
           end

data/lib/ood_core/job/adapters/linux_host.rb

@@ -0,0 +1,241 @@
+require "ood_core/refinements/hash_extensions"
+require "ood_core/refinements/array_extensions"
+require "ood_core/job/adapters/helper"
+require "set"
+
+module OodCore
+  module Job
+    class Factory
+      using Refinements::HashExtensions
+
+      # Build the LinuxHost adapter from a configuration
+      # @param config [#to_h] the configuration for job adapter
+      # @option config [Object] :contain (false) Pass `--contain` flag to Singularity; allows overriding bind mounts in singularity.conf
+      # @option config [Object] :debug (false) Use the adapter in a debug mode
+      # @option config [Object] :max_timeout (nil) The longest 'wall_clock' permissible
+      # @option config [Object] :singularity_bin ('/usr/bin/singularity') The path to the Singularity executable
+      # @option config [Object] :singularity_bindpath ('/etc,/media,/mnt,/opt,/srv,/usr,/var,/users') A comma delimited list of paths to bind between the host and the guest
+      # @option config [Object] :singularity_image The path to the Singularity image to use
+      # @option config [Object] :ssh_hosts (nil) The list of permissable hosts, defaults to :submit_host
+      # @option config [Object] :strict_host_checking (true) Set to false to disable strict host checking and updating the known_hosts file
+      # @option config [Object] :submit_host The SSH target to connect to, may be the head of a round-robin
+      # @option config [Object] :tmux_bin ('/usr/bin/tmux') The path to the Tmux executable
+      def self.build_linux_host(config)
+        c = config.to_h.symbolize_keys
+        contain = c.fetch(:contain, false)
+        debug = c.fetch(:debug, false)
+        max_timeout = c.fetch(:max_timeout, nil)
+        singularity_bin = c.fetch(:singularity_bin, '/usr/bin/singularity')
+        singularity_bindpath = c.fetch(:singularity_bindpath, '/etc,/media,/mnt,/opt,/srv,/usr,/var,/users')
+        singularity_image = c[:singularity_image]
+        ssh_hosts = c.fetch(:ssh_hosts, [c[:submit_host]])
+        strict_host_checking = c.fetch(:strict_host_checking, true)
+        submit_host = c[:submit_host]
+        tmux_bin = c.fetch(:tmux_bin, '/usr/bin/tmux')
+
+        Adapters::LinuxHost.new(
+          ssh_hosts: ssh_hosts,
+          launcher: Adapters::LinuxHost::Launcher.new(
+            contain: contain,
+            debug: debug,
+            max_timeout: max_timeout,
+            singularity_bin: singularity_bin,
+            singularity_bindpath: singularity_bindpath,  # '/etc,/media,/mnt,/opt,/srv,/usr,/var,/users',
+            singularity_image: singularity_image,
+            ssh_hosts: ssh_hosts,
+            strict_host_checking: strict_host_checking,
+            submit_host: submit_host,
+            tmux_bin: tmux_bin,
+          )
+        )
+      end
+    end
+
+    module Adapters
+      # An adapter object that describes the communication with a remote host
+      # for job management.
+      class LinuxHost < Adapter
+        using Refinements::ArrayExtensions
+
+        require "ood_core/job/adapters/linux_host/launcher"
+
+        def initialize(ssh_hosts:, launcher:)
+          @launcher = launcher
+          @ssh_hosts = Set.new(ssh_hosts)
+        end
+
+        # Submit a job with the attributes defined in the job template instance
+        # @param script [Script] script object that describes the script and
+        #   attributes for the submitted job
+        # @param after [#to_s, Array<#to_s>] No scheduling is available is used; setting raises JobAdapterError
+        # @param afterok [#to_s, Array<#to_s>] No scheduling is available is used; setting raises JobAdapterError
+        # @param afternotok [#to_s, Array<#to_s>] No scheduling is available is used; setting raises JobAdapterError
+        # @param afterany [#to_s, Array<#to_s>] No scheduling is available is used; setting raises JobAdapterError
+        # @raise [JobAdapterError] if something goes wrong submitting a job
+        # @return [String] the job id returned after successfully submitting a
+        #   job
+        # @see Adapter#submit
+        def submit(script, after: [], afterok: [], afternotok: [], afterany: [])
+          unless (after.empty? && afterok.empty? && afternotok.empty? && afterany.empty?)
+            raise JobAdapterError, 'Scheduling subsequent jobs is not available.'
+          end
+
+          @launcher.start_remote_session(script)
+        rescue Launcher::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Retrieve info for all jobs from the resource manager
+        # @raise [JobAdapterError] if something goes wrong getting job info
+        # @return [Array<Info>] information describing submitted jobs
+        # @see Adapter#info_all
+        def info_all(attrs: nil, host: nil)
+          host_permitted?(host) if host
+
+          @launcher.list_remote_sessions(host: host).map{
+            |ls_output| ls_to_info(ls_output)
+          }
+        rescue Launcher::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Retrieve info for all jobs for a given owner or owners from the
+        # resource manager
+        # Note: owner and attrs are present only to complete the interface and are ignored
+        # Note: since this API is used in production no errors or warnings are thrown / issued
+        # @param owner [#to_s, Array<#to_s>] the owner(s) of the jobs
+        # @raise [JobAdapterError] if something goes wrong getting job info
+        # @return [Array<Info>] information describing submitted jobs
+        def info_where_owner(owner: nil, attrs: nil)
+          info_all
+        end
+
+        # Iterate over each job Info object
+        # @param attrs [Array<symbol>] attrs is present only to complete the interface and is ignored
+        # @yield [Info] of each job to block
+        # @return [Enumerator] if no block given
+        def info_all_each(attrs: nil)
+          return to_enum(:info_all_each, attrs: attrs) unless block_given?
+
+          info_all(attrs: attrs).each do |job|
+            yield job
+          end
+        end
+
+        # Iterate over each job Info object
+        # @param owner [#to_s, Array<#to_s>] owner is present only to complete the interface and is ignored
+        # @param attrs [Array<symbol>] attrs is present only to complete the interface and is ignored
+        # @yield [Info] of each job to block
+        # @return [Enumerator] if no block given
+        def info_where_owner_each(owner, attrs: nil)
+          return to_enum(:info_where_owner_each, owner, attrs: attrs) unless block_given?
+
+          info_where_owner(owner, attrs: attrs).each do |job|
+            yield job
+          end
+        end
+
+        # Whether the adapter supports job arrays
+        # @return [Boolean] - false
+        def supports_job_arrays?
+          false
+        end
+
+        # Retrieve job info from the SSH host
+        # @param id [#to_s] the id of the job
+        # @raise [JobAdapterError] if something goes wrong getting job info
+        # @return [Info] information describing submitted job
+        # @see Adapter#info
+        def info(id)
+          _, host = parse_job_id(id)
+          job = info_all(host: host).select{|info| info.id == id}.first
+          (job) ? job : Info.new(id: id, status: :completed)
+        rescue Launcher::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Retrieve job status from resource manager
+        # @note Optimized slightly over retrieving complete job information from server
+        # @abstract Subclass is expected to implement {#status}
+        # @raise [NotImplementedError] if subclass did not define {#status}
+        # @param id [#to_s] the id of the job
+        # @return [Status] status of job
+        def status(id)
+          _, host = parse_job_id(id)
+          job = info_all(host: host).select{|info| info.id == id}.first
+
+          Status.new(state: (job) ? :running : :completed)
+        rescue Launcher::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        # Put the submitted job on hold
+        # @abstract Subclass is expected to implement {#hold}
+        # @raise [NotImplementedError] if subclass did not define {#hold}
+        # @param id [#to_s] the id of the job
+        # @return [void]
+        def hold(id)
+          # Consider sending SIGSTOP?
+          raise NotImplementedError, "subclass did not define #hold"
+        end
+
+        # Release the job that is on hold
+        # @abstract Subclass is expected to implement {#release}
+        # @raise [NotImplementedError] if subclass did not define {#release}
+        # @param id [#to_s] the id of the job
+        # @return [void]
+        def release(id)
+          # Consider sending SIGCONT
+          raise NotImplementedError, "subclass did not define #release"
+        end
+
+        # Delete the submitted job
+        # @abstract Subclass is expected to implement {#delete}
+        # @raise [NotImplementedError] if subclass did not define {#delete}
+        # @param id [#to_s] the id of the job
+        # @return [void]
+        def delete(id)
+          session_name, destination_host = parse_job_id(id)
+          @launcher.stop_remote_session(session_name, destination_host)
+        rescue Launcher::Error => e
+          raise JobAdapterError, e.message
+        end
+
+        private
+
+        def host_permitted?(destination_host)
+          raise JobAdapterError, "Requested destination host (#{destination_host}) not permitted" unless @ssh_hosts.include?(destination_host)
+        end
+
+        def parse_job_id(id)
+          raise JobAdapterError, "#{id} is not a valid LinuxHost adapter id because it is missing the '@'." unless id.include?('@')
+
+          return id.split('@')
+        end
+
+        # Convert the returned Hash into an Info object
+        def ls_to_info(ls_output)
+          started = ls_output[:session_created].to_i
+          now = Time.now.to_i
+          ellapsed = now - started
+          Info.new(
+              accounting_id: nil,
+              allocated_nodes: [NodeInfo.new(name: ls_output[:destination_host], procs: 1)],
+              cpu_time: ellapsed,
+              dispatch_time: started,
+              id: ls_output[:id],
+              job_name: nil,  # TODO
+              job_owner: Etc.getlogin,
+              native: ls_output,
+              procs: 1,
+              queue_name: "LinuxHost adapter for #{@submit_host}",
+              status: :running,
+              submission_time: ellapsed,
+              submit_host: @submit_host,
+              wallclock_time: ellapsed
+          )
+        end
+      end
+    end
+  end
+end

data/lib/ood_core/job/adapters/linux_host/launcher.rb

@@ -0,0 +1,262 @@
+require 'erb'
+require 'etc'
+require 'pathname'
+require 'securerandom'
+require 'shellwords'
+require 'time'
+
+# Object used for simplified communication SSH hosts
+#
+# @api private
+class OodCore::Job::Adapters::LinuxHost::Launcher
+  attr_reader :contain, :debug, :site_timeout, :session_name_label, :singularity_bin,
+    :site_singularity_bindpath, :default_singularity_image, :ssh_hosts,
+    :strict_host_checking, :submit_host, :tmux_bin, :username
+  # The root exception class that all LinuxHost adapter-specific exceptions inherit
+  # from
+  class Error < StandardError; end
+
+  UNIT_SEPARATOR = "\x1F"
+
+  # @param debug Whether the adapter should be used in debug mode
+  # @param site_timeout [#to_i] A period after which the job should be killed or nil
+  # @param singularity_bin  Path to the Singularity executable
+  # @param singularity_bindpath A comma delimited string of host paths to bindmount into the guest; sets SINGULARITY_BINDPATH environment variable
+  # @param singularity_image [#to_s] Path to the Singularity image
+  # @param ssh_hosts List of hosts to check when scanning for running jobs
+  # @param strict_host_checking Allow SSH to perform strict host checking
+  # @param submit_host The SSH-able host
+  # @param tmux_bin [#to_s] Path to the tmux executable
+  def initialize(
+    contain: false,
+    debug: false,
+    site_timeout: nil,
+    singularity_bin:,
+    singularity_bindpath: '/etc,/media,/mnt,/opt,/run,/srv,/usr,/var,/users',
+    singularity_image:,
+    ssh_hosts:,
+    strict_host_checking: false,
+    submit_host:,
+    tmux_bin:,
+    **_
+  )
+    @contain = !! contain
+    @debug = !! debug
+    @site_timeout = site_timeout.to_i
+    @session_name_label = 'launched-by-ondemand'
+    @singularity_bin = Pathname.new(singularity_bin)
+    @site_singularity_bindpath = singularity_bindpath.to_s
+    @default_singularity_image = Pathname.new(singularity_image)
+    @ssh_hosts = ssh_hosts
+    @strict_host_checking = strict_host_checking
+    @submit_host = submit_host
+    @tmux_bin = tmux_bin
+    @username = Etc.getlogin
+  end
+
+  # @param hostname [#to_s] The hostname to submit the work to
+  # @param script [OodCore::Job::Script] The script object defining the work
+  def start_remote_session(script)
+    cmd = ssh_cmd(submit_host)
+
+    session_name = unique_session_name
+    output = call(*cmd, stdin: wrapped_script(script, session_name))
+    hostname = output.strip
+
+    "#{session_name}@#{hostname}"
+  end
+
+  def stop_remote_session(session_name, hostname)
+    cmd = ssh_cmd(hostname)
+
+    kill_cmd = <<~SCRIPT
+    # Get the tmux pane PID for the target session
+    pane_pid=$(tmux list-panes -aF '\#{session_name} \#{pane_pid}' | grep '#{session_name}' | cut -f 2 -d ' ')
+    # Find the Singularity sinit PID child of the pane process
+    pane_sinit_pid=$(pstree -p "$pane_pid" | grep -o 'sinit([[:digit:]]*' | grep -o '[[:digit:]]*')
+    # Kill sinit which stops both Singularity-based processes and the tmux session
+    kill "$pane_sinit_pid"
+    SCRIPT
+
+    call(*cmd, stdin: kill_cmd)
+  rescue Error => e
+    raise e unless (
+      # The tmux server not running is not an error
+      e.message.include?('failed to connect to server') ||
+      # The session not being found is not an error
+      e.message.include?("session not found: #{session_name_label}")
+    )
+  end
+
+  def list_remote_sessions(host: nil)
+    host_list = (host) ? [host] : ssh_hosts
+
+    host_list.map {
+      |hostname| list_remote_tmux_session(hostname)
+    }.flatten.sort_by {
+      |hsh| hsh[:session_name]
+    }
+  end
+
+  private
+
+  # Call a forked Slurm command for a given cluster
+  def call(cmd, *args, env: {}, stdin: "")
+    args  = args.map(&:to_s)
+    env = env.to_h
+    o, e, s = Open3.capture3(env, cmd, *args, stdin_data: stdin.to_s)
+    s.success? ? o : raise(Error, e)
+  end
+
+  # The SSH invocation to send a command
+  # -t Force pseudo-terminal allocation (required to allow tmux to run)
+  # -o BatchMode=yes (set mode to be non-interactive)
+  # if ! strict_host_checking
+  # -o UserKnownHostsFile=/dev/null (do not update the user's known hosts file)
+  # -o StrictHostKeyChecking=no (do no check the user's known hosts file)
+  def ssh_cmd(destination_host)
+    if strict_host_checking
+      [
+        'ssh', '-t',
+        '-o', 'BatchMode=yes',
+        "#{username}@#{destination_host}"
+      ]
+    else
+      [
+        'ssh', '-t',
+        '-o', 'BatchMode=yes',
+        '-o', 'UserKnownHostsFile=/dev/null',
+        '-o', 'StrictHostKeyChecking=no',
+        "#{username}@#{destination_host}"
+      ]
+    end
+  end
+
+  def shell
+    ENV['SHELL'] || '/bin/bash'
+  end
+
+  # Wraps a user-provided script into a Tmux invocation
+  def wrapped_script(script, session_name)
+    content = script.content
+    unless user_script_has_shebang?(script)
+     content = "#!#{shell}\n#{content}"
+    end
+
+    ERB.new(
+      File.read(Pathname.new(__dir__).join('templates/script_wrapper.erb.sh'))
+    ).result(binding.tap {|bnd|
+      {
+        'arguments' => script_arguments(script),
+        'cd_to_workdir' => (script.workdir) ? "cd #{script.workdir}" : '',
+        'contain' => (contain) ? '--contain' : '',
+        'debug' => debug,
+        'email_on_terminated' => script_email_on_event(script, 'terminated'),
+        'email_on_start' => script_email_on_event(script, 'started'),
+        'environment' => export_env(script),
+        'error_path' => (script.error_path) ? script.error_path.to_s : '/dev/null',
+        'job_name' => script.job_name.to_s,
+        'output_path' => (script.output_path) ? script.output_path.to_s : '/dev/null',
+        'script_content' => content,
+        'script_timeout' => script_timeout(script),
+        'session_name' => session_name,
+        'singularity_bin' => singularity_bin,
+        'singularity_image' => singularity_image(script.native),
+        'tmux_bin' => tmux_bin,
+      }.each{
+        |key, value| bnd.local_variable_set(key, value)
+      }
+    })
+  end
+
+  # Generate the environment export block for this script
+  def export_env(script)
+    environment = script.job_environment
+    (environment ? environment : {}).tap{
+      |hsh|
+      hsh['SINGULARITY_BINDPATH'] = singularity_bindpath(script.native)
+    }.map{
+      |key, value| "export #{key}=#{Shellwords.escape(value)}"
+    }.sort.join("\n")
+  end
+
+  def singularity_image(native)
+    if native && native[:singularity_container]
+      return native[:singularity_container]
+    end
+
+    default_singularity_image
+  end
+
+  def singularity_bindpath(native)
+    return site_singularity_bindpath unless native && native[:singularity_bindpath]
+
+    native[:singularity_bindpath]
+  end
+
+  def script_timeout(script)
+    wall_time = script.wall_time.to_i
+    return site_timeout if wall_time == 0
+    return [wall_time, site_timeout].min unless site_timeout == 0
+
+    wall_time
+  end
+
+  def script_arguments(script)
+    return '' unless script.args
+
+    Shellwords.join(script.args)
+  end
+
+  def script_email_on_event(script, event)
+    return false unless script.email && script.send("email_on_#{event}")
+
+    ERB.new(
+      File.read(Pathname.new(__dir__).join('templates/email.erb.sh'))
+    ).result(binding.tap {|bnd|
+      {
+        'email_recipients' => script.email.map{|addr| Shellwords.escape(addr)}.join(', '),
+        'job_name' => (script.job_name) ? script.job_name : 'LinuxHost_Adapter_Job',
+        'job_status' => event
+      }.each{
+        |key, value| bnd.local_variable_set(key, value)
+      }
+    })
+  end
+
+  def unique_session_name
+    "#{session_name_label}-#{SecureRandom.uuid}"
+  end
+
+  # List all Tmux sessions on destination_host started by this adapter
+  # Additional tmux ls options available: http://man7.org/linux/man-pages/man1/tmux.1.html#FORMATS
+  def list_remote_tmux_session(destination_host)
+    # Note that the tmux variable substitution looks like Ruby string sub,
+    # these must either be single quoted strings or Ruby-string escaped as well
+    format_str = Shellwords.escape(
+      ['#{session_name}', '#{session_created}', '#{pane_pid}'].join(UNIT_SEPARATOR)
+    )
+    keys = [:session_name, :session_created, :session_pid]
+    cmd = ssh_cmd(destination_host) + ['tmux', 'list-panes', '-aF', format_str]
+
+    call(*cmd).split(
+      "\n"
+    ).map do |line|
+      Hash[keys.zip(line.split(UNIT_SEPARATOR))].tap do |session_hash|
+        session_hash[:destination_host] = destination_host
+        session_hash[:id] = "#{session_hash[:session_name]}@#{destination_host}"
+      end
+    end.select{
+      |session_hash| session_hash[:session_name].start_with?(session_name_label)
+    }
+  rescue Error => e
+    # The tmux server not running is not an error
+    raise e unless e.message.include?('failed to connect to server')
+    []
+  end
+
+  def user_script_has_shebang?(script)
+    return false if script.content.empty?
+    script.content.split("\n").first.start_with?('#!/')
+  end
+end

data/lib/ood_core/job/adapters/linux_host/templates/email.erb.sh

@@ -0,0 +1,9 @@
+if command -v mail; then
+cat << EMAIL_CONTENT | mail -s "Job <%= job_name %> has <%= job_status %>" <%= email_recipients %>
+Greetings,
+
+Your job <%= job_name %> has <%= job_status %>.
+
+- The OnDemand Linux Host Adapter
+EMAIL_CONTENT
+fi

data/lib/ood_core/job/adapters/linux_host/templates/script_wrapper.erb.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+hostname
+
+# Put the script into a temp file on localhost
+<% if debug %>
+singularity_tmp_file=$(mktemp -p "$HOME" --suffix '_sing')
+tmux_tmp_file=$(mktemp -p "$HOME" --suffix "_tmux")
+<% else %>
+singularity_tmp_file=$(mktemp)
+tmux_tmp_file=$(mktemp)
+<% end %>
+
+# Create an executable to run in a tmux session
+# The escaped HEREDOC means that we need to substitute in $singularity_tmp_file ourselves
+cat << 'TMUX_LAUNCHER' | sed "s#\$singularity_tmp_file#${singularity_tmp_file}#" > "$tmux_tmp_file"
+#!/bin/bash
+<% if email_on_terminated %>
+exit_script() {
+<%# DO NOT INDENT email_on_terminated may have HEREDOCS %>
+<%= email_on_terminated %>
+trap - SIGINT SIGTERM # clear the trap
+kill -- -$$ # Sends SIGTERM to child/sub processes
+}
+trap exit_script SIGINT SIGTERM
+<% end %>
+
+<%= cd_to_workdir %>
+<%= environment %>
+
+<%= email_on_start %>
+
+# Redirect stdout and stderr to separate files for all commands run within the curly braces
+# https://unix.stackexchange.com/a/6431/204548
+# Swap sterr and stdout after stdout has been redirected
+# https://unix.stackexchange.com/a/61932/204548
+OUTPUT_PATH=<%= output_path %>
+ERROR_PATH=<%= error_path %>
+({
+timeout <%= script_timeout %>s <%= singularity_bin %> exec <%= contain %> --pid <%= singularity_image %> /bin/bash --login $singularity_tmp_file <%= arguments %>
+} | tee "$OUTPUT_PATH") 3>&1 1>&2 2>&3 | tee "$ERROR_PATH"
+
+<%= email_on_terminated %>
+
+# Exit the tmux session when we are complete
+exit 0
+TMUX_LAUNCHER
+
+# Create an executable for Singularity to run
+# Escaped HEREDOC means that we do not have to worry about Shell.escape-ing script_content
+cat << 'SINGULARITY_LAUNCHER' > "$singularity_tmp_file"
+<%= script_content %>
+SINGULARITY_LAUNCHER
+
+# Run the script inside a tmux session
+chmod +x "$singularity_tmp_file"
+chmod +x "$tmux_tmp_file"
+<%= tmux_bin %> new-session -d -s "<%= session_name %>" "$tmux_tmp_file"
+
+# Remove the file
+<% if ! debug %>
+# Wait 1 second to ensure that tmux session has started before the file is removed
+sleep 1
+rm -f "$tmux_tmp_file"; rm -f "$singularity_tmp_file"
+<% end %>

data/lib/ood_core/version.rb

@@ -1,4 +1,4 @@
 module OodCore
   # The current version of {OodCore}
-  VERSION = "0.9.3"
+  VERSION = "0.10.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ood_core
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.10.0
 platform: ruby
 authors:
 - Eric Franz
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-05-09 00:00:00.000000000 Z
+date: 2019-11-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ood_support
@@ -150,6 +150,10 @@ files:
 - lib/ood_core/job/adapter.rb
 - lib/ood_core/job/adapters/drmaa.rb
 - lib/ood_core/job/adapters/helper.rb
+- lib/ood_core/job/adapters/linux_host.rb
+- lib/ood_core/job/adapters/linux_host/launcher.rb
+- lib/ood_core/job/adapters/linux_host/templates/email.erb.sh
+- lib/ood_core/job/adapters/linux_host/templates/script_wrapper.erb.sh
 - lib/ood_core/job/adapters/lsf.rb
 - lib/ood_core/job/adapters/lsf/batch.rb
 - lib/ood_core/job/adapters/lsf/helper.rb
@@ -197,7 +201,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.14
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Open OnDemand core library