scout-gear 10.8.4 → 10.10.1

data/lib/scout/workflow/deployment/scheduler/lfs.rb

@@ -0,0 +1,125 @@
+require_relative 'job'
+require 'scout'
+
+module LSF 
+  extend SchedulerJob
+
+  def self.system
+    "LSF"
+  end
+
+  def self.batch_system_variables
+    <<-EOF
+let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
+let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $LSB_MAX_NUM_PROCESSORS )"
+[ ! -z $LSB_MAX_MEM_RUSAGE ] && let MAX_MEMORY="$LSB_MAX_MEM_RUSAGE" || MAX_MEMORY="$MAX_MEMORY_DEFAULT"
+export MAX_MEMORY_DEFAULT
+export MAX_MEMORY
+export BATCH_JOB_ID=$LSF_JOBID
+export BATCH_SYSTEM=#{system}
+    EOF
+  end
+
+  def self.header(options = {})
+    options        = options.dup
+
+    queue          = IndiferentHash.process_options options, :queue
+    task_cpus      = IndiferentHash.process_options options, :task_cpus
+    time           = IndiferentHash.process_options options, :time
+    nodes          = IndiferentHash.process_options options, :nodes
+    workdir        = IndiferentHash.process_options options, :workdir
+    exclusive      = IndiferentHash.process_options options, :exclusive
+
+    batch_dir  = IndiferentHash.process_options options, :batch_dir
+    batch_name = IndiferentHash.process_options options, :batch_name
+    batch_name ||= File.basename(batch_dir)
+
+    fout       = File.join(batch_dir, 'std.out')
+    ferr       = File.join(batch_dir, 'std.err')
+
+    time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
+
+    time = time.split(":").values_at(0, 1) * ":"
+
+    header =<<-EOF
+#!/bin/bash
+#BSUB -J "#{batch_name}"
+#BSUB -cwd "#{workdir}"
+#BSUB -oo "#{fout}"
+#BSUB -eo "#{ferr}"
+#BSUB -q "#{queue}"
+#BSUB -n "#{task_cpus}"
+#BSUB -W "#{time}"
+    EOF
+
+    header << "#BSUB -x" << "\n" if exclusive
+
+    header
+  end
+
+  def self.run_template(batch_dir, dry_run)
+
+    fout   = File.join(batch_dir, 'std.out')
+    ferr   = File.join(batch_dir, 'std.err')
+    fjob   = File.join(batch_dir, 'job.id')
+    fdep   = File.join(batch_dir, 'dependencies.list')
+    fcfdep = File.join(batch_dir, 'canfail_dependencies.list')
+    fexit  = File.join(batch_dir, 'exit.status')
+    fsync  = File.join(batch_dir, 'sync.log')
+    fcmd   = File.join(batch_dir, 'command.batch')
+
+    return if Open.exists?(fexit)
+
+    STDERR.puts Log.color(:magenta, "Issuing LSF file: #{fcmd}")
+    STDERR.puts Open.read(fcmd)
+
+    if File.exist?(fjob)
+      job = Open.read(fjob).to_i
+    else
+
+      dependencies = Open.read(fdep).split("\n") if File.exist? fdep
+      canfail_dependencies = Open.read(fcfdep).split("\n") if File.exist? fcfdep
+
+      normal_dep_list = dependencies && dependencies.any? ? dependencies.collect{|d| "post_done(#{d})"} : []
+      canfail_dep_list = canfail_dependencies && canfail_dependencies.any? ? canfail_dependencies.collect{|d| "done(#{d})"} : []
+
+      dep_list = normal_dep_list + canfail_dep_list
+
+      if dep_list.any?
+        dep_str = '-w "' + dep_list * " && " + '"'
+      else
+        dep_str = ""
+      end
+
+      cmd = "bsub #{dep_str} < '#{fcmd}'"
+
+      if File.exist?(fout)
+        return
+      elsif dry_run
+        STDERR.puts Log.color(:magenta, "To execute run: ") + Log.color(:blue, cmd)
+        STDERR.puts Log.color(:magenta, "To monitor progress run (needs local scout): ") + Log.color(:blue, "scout lsf tail '#{batch_dir}'")
+        raise DryRun, batch_dir
+      else
+        Open.rm fsync
+        Open.rm fexit
+        Open.rm fout
+        Open.rm ferr
+
+
+        job = CMD.cmd(cmd).read.scan(/\d+/).first.to_i
+        Log.debug "BSUB job id: #{job}"
+        Open.write(fjob, job.to_s)
+        job
+      end
+    end
+  end
+
+  def self.job_status(job = nil)
+    if job.nil?
+      CMD.cmd("bjobs -w").read
+    else
+      CMD.cmd("bjobs -w #{job}").read
+    end
+  end
+end
+

data/lib/scout/workflow/deployment/scheduler/pbs.rb

@@ -0,0 +1,176 @@
+require_relative 'job'
+require 'scout'
+
+module PBS 
+  extend SchedulerJob
+
+  def self.system
+    "PBS"
+  end
+
+  def self.batch_system_variables
+    <<-EOF
+let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
+let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $PBS_CPUS_PER_TASK )"
+MAX_MEMORY="$MAX_MEMORY_DEFAULT"
+[ ! -z $PBS_MEM_PER_CPU ] && let MAX_MEMORY="$PBS_MEM_PER_CPU * $PBS_CPUS_PER_TASK" 
+[ ! -z $PBS_MEM_PER_NODE ] && MAX_MEMORY="$PBS_MEM_PER_NODE"
+export MAX_MEMORY_DEFAULT
+export MAX_MEMORY
+export BATCH_JOB_ID=$PBS_JOBID
+export BATCH_SYSTEM=#{system}
+
+cd ${PBS_O_WORKDIR}
+    EOF
+  end
+
+  def self.header(options = {})
+    options = options.dup
+
+    workdir    = IndiferentHash.process_options options, :workdir
+    batch_dir  = IndiferentHash.process_options options, :batch_dir
+    batch_name = IndiferentHash.process_options options, :batch_name
+
+    queue      = IndiferentHash.process_options options, :queue
+    account    = IndiferentHash.process_options options, :account
+    time       = IndiferentHash.process_options options, :time
+    nodes      = IndiferentHash.process_options options, :nodes
+
+    # PBS 
+    place      = IndiferentHash.process_options options, :place, :place => 'scatter'
+    system     = IndiferentHash.process_options options, :partition
+    filesystems = IndiferentHash.process_options options, :filesystems
+
+    filesystems = "home" if filesystems.nil?
+
+    filesystems = filesystems * "," if Array === filesystems
+
+    # NOT USED 
+    partition  = IndiferentHash.process_options options, :partition
+    task_cpus  = IndiferentHash.process_options options, :task_cpus
+    exclusive  = IndiferentHash.process_options options, :exclusive
+    highmem    = IndiferentHash.process_options options, :highmem
+    licenses   = IndiferentHash.process_options options, :licenses
+    constraint = IndiferentHash.process_options options, :constraint
+    gres       = IndiferentHash.process_options options, :gres
+
+    constraint     = [constraint, "highmem"].compact * "&" if highmem
+
+    mem            = IndiferentHash.process_options options, :mem
+    mem_per_cpu    = IndiferentHash.process_options options, :mem_per_cpu
+
+    fout       = File.join(batch_dir, 'std.out')
+    ferr       = File.join(batch_dir, 'std.err')
+
+    time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
+
+    qsub_params = {  "-l filesystems=" => filesystems,
+                     "-l system=" => system,
+                     "-l select=" => nodes,
+                     "-l place=" => place,
+                     "-l walltime=" => time,
+                     "-q " => queue,
+                     "-A " => account,
+                     "-o " => fout,
+                     "-e " => ferr,
+                     "-k doe" => true,
+                     # "cpus-per-task" => task_cpus,
+                     # "nodes" => nodes,
+                     # "time" => time,
+                     # "constraint" => constraint,
+                     # "exclusive" => exclusive,
+                     # "licenses" => licenses,
+                     # "gres" => gres,
+                     # "mem" => mem,
+                     # "mem-per-cpu" => mem_per_cpu,
+    }
+
+
+    header =<<-EOF
+#!/bin/bash
+    EOF
+
+    qsub_params.each do |name,value|
+      next if value.nil? || value == ""
+      if TrueClass === value
+        header << "#PBS #{name}" << "\n"
+      elsif Array === value
+        value.each do |v|
+          header << "#PBS #{name}\"#{v}\"" << "\n"
+        end
+      else
+        header << "#PBS #{name}\"#{value}\"" << "\n"
+      end
+    end
+
+    header
+  end
+
+  def self.run_template(batch_dir, dry_run)
+
+    fout   = File.join(batch_dir, 'std.out')
+    ferr   = File.join(batch_dir, 'std.err')
+    fjob   = File.join(batch_dir, 'job.id')
+    fdep   = File.join(batch_dir, 'dependencies.list')
+    fcfdep = File.join(batch_dir, 'canfail_dependencies.list')
+    fexit  = File.join(batch_dir, 'exit.status')
+    fsync  = File.join(batch_dir, 'sync.log')
+    fcmd   = File.join(batch_dir, 'command.batch')
+
+    return if Open.exists?(fexit)
+
+    Log.info "Issuing PBS file: #{fcmd}"
+    Log.debug Open.read(fcmd)
+
+    if File.exist?(fjob)
+      job = Open.read(fjob).to_i
+    else
+
+      dependencies = Open.read(fdep).split("\n") if File.exist? fdep
+      canfail_dependencies = Open.read(fcfdep).split("\n") if File.exist? fcfdep
+
+      normal_dep_str = dependencies && dependencies.any? ? "afterok:" + dependencies * ":" : nil
+      canfail_dep_str = canfail_dependencies && canfail_dependencies.any? ? "afterany:" + canfail_dependencies * ":" : nil
+
+      if normal_dep_str.nil? && canfail_dep_str.nil?
+        dep_str = ""
+      else
+        dep_str = '-W depend=' + [normal_dep_str, canfail_dep_str].compact * ","
+      end
+
+      cmd = "qsub #{dep_str} '#{fcmd}'"
+
+      if File.exist?(fout)
+        return
+      elsif dry_run
+        STDERR.puts Log.color(:magenta, "To execute run: ") + Log.color(:blue, "squb '#{fcmd}'")
+        STDERR.puts Log.color(:magenta, "To monitor progress run (needs local rbbt): ") + Log.color(:blue, "rbbt pbs tail '#{batch_dir}'")
+        raise DryRun, batch_dir
+      else
+        Open.rm fsync
+        Open.rm fexit
+        Open.rm fout
+        Open.rm ferr
+
+        job = CMD.cmd(cmd).read.scan(/\d+/).first.to_i
+        Log.debug "SBATCH job id: #{job}"
+        Open.write(fjob, job.to_s)
+        job
+      end
+    end
+  end
+
+  def self.job_status(job = nil)
+    if job.nil?
+      CMD.cmd("qstat").read
+    else
+      begin
+        CMD.cmd("qstat #{job}").read
+      rescue
+        ""
+      end
+    end
+  end
+
+end
+

data/lib/scout/workflow/deployment/scheduler/slurm.rb

@@ -0,0 +1,158 @@
+require_relative 'job'
+require 'scout'
+
+module SLURM 
+
+  extend SchedulerJob
+
+  def self.system
+    "SLURM"
+  end
+
+
+  def self.batch_system_variables
+    <<-EOF
+let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
+let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $SLURM_CPUS_PER_TASK )"
+MAX_MEMORY="$MAX_MEMORY_DEFAULT"
+[ ! -z $SLURM_MEM_PER_CPU ] && let MAX_MEMORY="$SLURM_MEM_PER_CPU * $SLURM_CPUS_PER_TASK" 
+[ ! -z $SLURM_MEM_PER_NODE ] && MAX_MEMORY="$SLURM_MEM_PER_NODE"
+export MAX_MEMORY_DEFAULT
+export MAX_MEMORY
+export BATCH_JOB_ID=$SLURM_JOB_ID
+export BATCH_SYSTEM=#{system}
+    EOF
+  end
+
+  def self.header(options = {})
+    options = options.dup
+
+    queue      = IndiferentHash.process_options options, :queue
+    account    = IndiferentHash.process_options options, :account
+    partition  = IndiferentHash.process_options options, :partition
+    task_cpus  = IndiferentHash.process_options options, :task_cpus
+    time       = IndiferentHash.process_options options, :time
+    nodes      = IndiferentHash.process_options options, :nodes
+    workdir    = IndiferentHash.process_options options, :workdir
+    exclusive  = IndiferentHash.process_options options, :exclusive
+    highmem    = IndiferentHash.process_options options, :highmem
+    licenses   = IndiferentHash.process_options options, :licenses
+    constraints = IndiferentHash.process_options options, :constraints
+    gres       = IndiferentHash.process_options options, :gres
+
+    constraints     = [constraints, "highmem"].compact * "&" if highmem
+
+    mem            = IndiferentHash.process_options options, :mem
+    mem_per_cpu    = IndiferentHash.process_options options, :mem_per_cpu
+
+    batch_dir  = IndiferentHash.process_options options, :batch_dir
+    batch_name = IndiferentHash.process_options options, :batch_name
+
+    fout       = File.join(batch_dir, 'std.out')
+    ferr       = File.join(batch_dir, 'std.err')
+
+    time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
+
+    sbatch_params = {"job-name" => batch_name,
+                     "qos" => queue,
+                     "account" => account,
+                     "partition" => partition,
+                     "output" => fout,
+                     "error" => ferr,
+                     "cpus-per-task" => task_cpus,
+                     "nodes" => nodes,
+                     "time" => time,
+                     "constraints" => constraints,
+                     "exclusive" => exclusive,
+                     "licenses" => licenses,
+                     "gres" => gres,
+                     "mem" => mem,
+                     "mem-per-cpu" => mem_per_cpu,
+    }
+
+
+    header =<<-EOF
+#!/bin/bash
+    EOF
+
+    sbatch_params.each do |name,value|
+      next if value.nil? || value == ""
+      if TrueClass === value
+        header << "#SBATCH --#{name}" << "\n"
+      elsif Array === value
+        value.each do |v|
+          header << "#SBATCH --#{name}=\"#{v}\"" << "\n"
+        end
+      else
+        header << "#SBATCH --#{name}=\"#{value}\"" << "\n"
+      end
+    end
+
+    header
+  end
+
+  def self.run_template(batch_dir, dry_run)
+
+    fout   = File.join(batch_dir, 'std.out')
+    ferr   = File.join(batch_dir, 'std.err')
+    fjob   = File.join(batch_dir, 'job.id')
+    fdep   = File.join(batch_dir, 'dependencies.list')
+    fcfdep = File.join(batch_dir, 'canfail_dependencies.list')
+    fexit  = File.join(batch_dir, 'exit.status')
+    fsync  = File.join(batch_dir, 'sync.log')
+    fcmd   = File.join(batch_dir, 'command.batch')
+
+    return if Open.exists?(fexit)
+
+    Log.info "Issuing SLURM file: #{fcmd}"
+
+    if File.exist?(fjob)
+      job = Open.read(fjob).to_i
+    else
+
+      dependencies = Open.read(fdep).split("\n") if File.exist? fdep
+      canfail_dependencies = Open.read(fcfdep).split("\n") if File.exist? fcfdep
+
+      normal_dep_str = dependencies && dependencies.any? ? "afterok:" + dependencies * ":" : nil
+      canfail_dep_str = canfail_dependencies && canfail_dependencies.any? ? "afterany:" + canfail_dependencies * ":" : nil
+
+      if normal_dep_str.nil? && canfail_dep_str.nil?
+        dep_str = ""
+      else
+        dep_str = '--dependency=' + [normal_dep_str, canfail_dep_str].compact * ","
+      end
+
+      cmd = "sbatch #{dep_str} '#{fcmd}'"
+
+      if File.exist?(fout)
+        return
+      elsif dry_run
+        STDERR.puts Log.color(:magenta, "To execute run: ") + Log.color(:blue, "sbatch '#{fcmd}'")
+        STDERR.puts Log.color(:magenta, "To monitor progress run (needs local scout): ") + Log.color(:blue, "scout slurm tail '#{batch_dir}'")
+        raise DryRun, batch_dir
+      else
+        Open.rm fsync
+        Open.rm fexit
+        Open.rm fout
+        Open.rm ferr
+
+        job = CMD.cmd(cmd).read.scan(/\d+/).first.to_i
+        Log.debug "SBATCH job id: #{job}"
+        Open.write(fjob, job.to_s)
+        job
+      end
+    end
+  end
+
+  def self.job_status(job = nil)
+    if job.nil?
+      CMD.cmd("squeue").read
+    else
+      begin
+        CMD.cmd("squeue --job #{job}").read
+      rescue
+        ""
+      end
+    end
+  end
+end

data/lib/scout/workflow/deployment/scheduler.rb

@@ -0,0 +1,73 @@
+require_relative 'orchestrator/chains'
+require_relative 'orchestrator/rules'
+require_relative 'orchestrator/batches'
+
+require_relative 'scheduler/slurm'
+require_relative 'scheduler/pbs'
+require_relative 'scheduler/lfs'
+
+module Workflow::Scheduler
+  def self.produce(jobs, rules = {}, options = {})
+    batches = Workflow::Orchestrator.job_batches(rules, jobs)
+    Workflow::Scheduler.process_batches(batches, options)
+  end
+
+  def self.process_batches(batches, process_options = {})
+    failed_jobs = []
+
+    pending = batches.dup
+
+    sorted = []
+    while pending.any?
+      leaf_nodes = batches.select{|batch| (batch[:deps] - sorted).empty? }
+      sorted.concat(leaf_nodes - sorted)
+      pending -= leaf_nodes
+    end
+
+    batch_system = Scout::Config.get :system, :batch, :scheduler, 'env:BATCH_SYSTEM', default: 'SLURM'
+
+    batch_ids = {}
+    sorted.collect do |batch|
+      job_options = batch[:rules]
+      job_options = IndiferentHash.add_defaults job_options, process_options.dup
+
+      if batch[:deps].nil?
+        batch_dependencies = [] 
+      else 
+        top_jobs = batch[:jobs]
+
+        batch_dependencies = batch[:deps].collect{|dep| 
+          dep_target = dep[:top_level]
+          id = batch_ids[dep_target].to_s
+
+          if dep_target.canfail?
+            'canfail:' + id
+          else
+            id
+          end
+        }
+      end
+
+      job_options.merge!(:batch_dependencies => batch_dependencies )
+      job_options.merge!(:manifest => batch[:jobs].collect{|d| d.task_signature })
+
+      begin
+        id, dir = case batch_system
+             when 'SLURM'
+               SLURM.run_job(batch[:top_level], job_options)
+             when 'LSF'
+               LSF.run_job(batch[:top_level], job_options)
+             when 'PBS'
+               PBS.run_job(batch[:top_level], job_options)
+             when nil
+               raise "No batch system specified"
+             else
+               raise "Unknown batch system #{batch_system}"
+             end
+        batch_ids[batch[:top_level]] = id
+      rescue DryRun
+        $!.message
+      end
+    end
+  end
+end

data/lib/scout/workflow/deployment.rb

@@ -1,2 +1,11 @@
-require_relative 'deployment/orchestrator'
+require_relative 'deployment/local'
+require_relative 'deployment/scheduler'
 require_relative 'deployment/trace'
+require_relative 'deployment/queue'
+
+module Workflow
+  def self.produce(jobs, ...)
+    rules = Workflow::Orchestrator.load_rules_for_job(jobs)
+    Workflow::LocalExecutor.produce(jobs, rules, ...)
+  end
+end

data/lib/scout/workflow/entity.rb

@@ -71,10 +71,31 @@ module EntityWorkflow
     end
 
     property_name = task_name.to_s.sub(/^(#{entity_name}_list|#{entity_name}|list)_/, '')
+    property_job_name = property_name + '_job'
+
+    property property_job_name => property_type do |*args|
+      job(task_name, *args)
+    end
+
     property property_name => property_type do |*args|
-      job = job(task_name, *args)
+      job = self.send(property_job_name)
+
+      job.join if job.running?
+
+      if job.error?
+        if job.recoverable_error?
+          job.clean
+        else
+          raise job.exception
+        end
+      end
+
+      job.run unless job.done?
+
+      job.load
       Array === job ? job.collect(&:run) : job.run
     end
+
   end
 
   def entity_task(task_name, *args, &block)

data/lib/scout/workflow/exceptions.rb

@@ -1 +1,3 @@
 class TaskNotFound < StandardError; end
+class DryRun < Exception; end
+

data/lib/scout/workflow/step/config.rb

@@ -8,10 +8,13 @@ class Step
     new_tokens = []
     if workflow
       workflow_name = workflow.name
-      new_tokens << ("workflow:" << workflow_name)
-      new_tokens << ("task:" << workflow_name << "#" << task_name.to_s)
+      new_tokens << ("workflow:" + workflow_name)
+      new_tokens << ("task:" + workflow_name << "#" << task_name.to_s)
     end
-    new_tokens << ("task:" << task_name.to_s)
+    new_tokens << ("task:" + task_name.to_s)
+    new_tokens << (task_name.to_s)
+    new_tokens << (workflow_name)
+    new_tokens << ("task")
 
     Scout::Config.get(key, tokens + new_tokens, options)
   end

data/lib/scout/workflow/step/file.rb

@@ -13,6 +13,10 @@ class Step
                    end
   end
 
+  def files_dir=(dir)
+    @files_dir = dir
+  end
+
   def file(file = nil)
     dir = files_dir
     Path.setup(dir) unless Path === dir

data/lib/scout/workflow/step/info.rb

@@ -38,8 +38,8 @@ class Step
     save_info(@info = info)
   end
 
-  def init_info
-    log :waiting unless info_file.nil? || Open.exists?(info_file)
+  def init_info(status=:waiting)
+    log status unless info_file.nil? || Open.exists?(info_file)
   end
 
   def info
@@ -120,7 +120,7 @@ class Step
       if info.include?(key)
         case info[key]
         when Array
-          info[key].concat Array === value ? value : [value]
+          info[key].concat(Array === value ? value : [value])
         when Hash
           info[key].merge! value
         else
@@ -201,7 +201,13 @@ class Step
   end
 
   def exception
-    info[:exception]
+    return nil unless info[:exception]
+    begin
+      Marshal.load(Base64.decode64(info[:exception]))
+    rescue
+      Log.exception $!
+      nil
+    end
   end
 
   # Marshal Step