scout-gear 10.11.4 → 10.11.7

data/lib/scout/workflow/task.rb

@@ -44,10 +44,23 @@ module Task
 
     memory_inputs = nil if Array === memory_inputs && memory_inputs.compact.empty?
     memory_inputs = nil if Hash === memory_inputs && memory_inputs.empty?
-    Persist.memory("Task job #{self.name}", repo: Workflow.job_cache, other: {task: self.name, id: id, provided_inputs: memory_inputs}) do
+    Persist.memory("Task job #{self.name}", repo: Workflow.job_cache, other: {workflow: self.workflow, task: self.name, id: id, provided_inputs: memory_inputs}) do
+
+      #{{{ Provided inputs and ID
       provided_inputs, id = id, nil if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
       provided_inputs = {} if provided_inputs.nil?
-      IndiferentHash.setup(provided_inputs)
+      IndiferentHash.setup(provided_inputs) if Hash === provided_inputs
+
+      provided_inputs = load_inputs(provided_inputs.delete(:load_inputs)).merge(provided_inputs) if Hash === provided_inputs && provided_inputs[:load_inputs]
+
+      provided_input_names = case provided_inputs
+                             when nil
+                               []
+                             when Array
+                               inputs.collect{|name,*| name }[0..provided_inputs.length]
+                             when Hash
+                               provided_inputs.keys
+                             end
 
       jobname_input = nil
       inputs.each do |name,type,desc,default,input_options|
@@ -56,12 +69,14 @@ module Task
       end
 
       id = provided_inputs[jobname_input] if jobname_input && id.nil?
-      #id = provided_inputs[:id] if provided_inputs.include?(:id)
+      id = DEFAULT_NAME if id.nil?
+      id = Path.sanitize_filename(id, 150)
 
+      #{{{ Missing inputs
       missing_inputs = []
       self.inputs.each do |input,type,desc,val,options|
         next unless options && options[:required]
-        missing_inputs << input unless provided_inputs.include?(input)
+        missing_inputs << input unless provided_input_names.include?(input.to_s) || provided_input_names.include?(input.to_sym)
       end if self.inputs
 
       if missing_inputs.length == 1
@@ -72,25 +87,32 @@ module Task
         raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
       end
 
-      provided_inputs = load_inputs(provided_inputs.delete(:load_inputs)).merge(provided_inputs) if Hash === provided_inputs && provided_inputs[:load_inputs]
-
+      #{{{ Process inputs
       job_inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
+      non_default_inputs.uniq!
+      NamedArray.setup(job_inputs, @inputs.collect{|i| i[0] }) if @inputs
+      step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
 
+      #{{{ Dependencies
       compute = {}
       dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
 
-      non_default_inputs.uniq!
-
-      non_default_inputs.delete_if{|k| k.to_s.include? "#" } unless dependencies.select{|d| d.overriden? }.any?
+      #{{{ Overrides
+      override_inputs = provided_input_names.select{|k| (String === k) && k.include?("#")  }
+      overriden = override_inputs.any? && dependencies.select{|dep| dep.overrider? || dep.overriden? }.any?
 
-      id = DEFAULT_NAME if id.nil?
+      non_default_inputs.delete_if{|input| override_inputs.include? input }
+      non_default_provided_inputs = non_default_inputs & provided_input_names
+      non_default_provided_inputs.delete jobname_input if provided_inputs[jobname_input] == id
 
-      sanitized_id = Path.sanitize_filename(id, 150)
-      if non_default_inputs.any? && !(non_default_inputs == [jobname_input] && provided_inputs[jobname_input] == id)
+      #{{{ Hash and Path
+      if overriden || non_default_provided_inputs.any?
         hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
-        name = [sanitized_id, hash] * "_"
+        Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies, overriden: override_inputs)}"
+        name = [id, hash] * "_"
       else
-        name = sanitized_id
+        Log.debug "ID #{self.name} #{id} - Clean"
+        name = id
       end
 
       extension = self.extension
@@ -110,17 +132,10 @@ module Task
       end
 
       path = directory[name]
-
       path = path.set_extension(extension) if extension
 
-      if hash
-        Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
-      else
-        Log.debug "ID #{self.name} #{id} - Clean"
-      end
-      NamedArray.setup(job_inputs, @inputs.collect{|i| i[0] }) if @inputs
       step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
-      Step.new path.find, job_inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
+      Step.new path.find, job_inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, overriden, &self
     end
   end
 

data/lib/scout/workflow/usage.rb

@@ -5,19 +5,19 @@ module Task
     str = StringIO.new
 
     if description
-      title, paragraph = description.split("\n\n")
+      title, _, paragraph = description.partition("\n\n")
       if title.length < Misc::MAX_TTY_LINE_WIDTH
         title = self.name.to_s + " - " + title
         str.puts Log.color :yellow, title
         str.puts Log.color :yellow, "-" * title.length
         if paragraph
-          str.puts "\n" << Misc.format_paragraph(paragraph) 
+          str.puts "\n" << Misc.format_paragraph(paragraph)
         end
       else
         title = self.name.to_s
         str.puts Log.color :yellow, title
         str.puts Log.color :yellow, "-" * title.length
-        str.puts "\n" << Misc.format_paragraph(description) 
+        str.puts "\n" << Misc.format_paragraph(description)
       end
     else
       title = self.name.to_s
@@ -92,7 +92,7 @@ module Task
       str.puts
       str.puts Log.color(:magenta, "Input select options")
       selects.collect{|p| p}.uniq.each do |input,options|
-        str.puts 
+        str.puts
         str.puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
       end
     end
@@ -164,7 +164,7 @@ module Workflow
   end
 
   def _prov_tasks(tree)
-    tasks = [] 
+    tasks = []
     heap = tree.values
     heap = [tree]
     while heap.any?
@@ -199,7 +199,7 @@ module Workflow
       else
         description += ";" + task_name.to_s
       end
-      
+
       seen << [workflow, task_name]
     end
     description
@@ -212,9 +212,9 @@ module Workflow
 
     offset_str = " " * offset
 
-    lines << offset_str 
+    lines << offset_str
 
-    tree.each do |p,dtree| 
+    tree.each do |p,dtree|
       next if seen.include?(p)
       seen.push(p)
       workflow, task = p
@@ -233,7 +233,7 @@ module Workflow
       str.puts Log.color :magenta, title
       str.puts Log.color :magenta, "=" * title.length
     else
-      str.puts Log.color :magenta, self.name 
+      str.puts Log.color :magenta, self.name
       str.puts Log.color :magenta, "=" * self.name.length
     end
 
@@ -244,14 +244,14 @@ module Workflow
     elsif task.nil?
 
       if self.documentation[:description] and not self.documentation[:description].empty?
-        str.puts Misc.format_paragraph self.documentation[:description] 
+        str.puts Misc.format_paragraph self.documentation[:description]
         str.puts
       end
 
       str.puts Log.color :magenta, "## TASKS"
       if self.documentation[:task_description] and not self.documentation[:task_description].empty?
         str.puts
-        str.puts Misc.format_paragraph self.documentation[:task_description] 
+        str.puts Misc.format_paragraph self.documentation[:task_description]
       end
       str.puts
 
@@ -276,7 +276,7 @@ module Workflow
 
         prov_string = prov_string(dep_tree(name))
         str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
-      end 
+      end
 
     else
 
@@ -302,7 +302,7 @@ module Workflow
               offset, workflow, task_name =  m.values_at 1, 2, 3
               str.puts [offset, Log.color(:magenta, workflow), "#", Log.color(:yellow, task_name)] * ""
           else
-            str.puts Log.color :blue, line 
+            str.puts Log.color :blue, line
           end
         end
         str.puts

data/lib/scout/workflow/util.rb

@@ -18,7 +18,7 @@ module Workflow
   end
 
   def all_tasks
-    tasks.keys
+    tasks.nil? ? [] : tasks.keys
   end
 
   def self.list

data/lib/scout/workflow.rb

@@ -27,9 +27,9 @@ module Workflow
     end
 
     def workflow_dir
-      @workflow_dir || 
-        ENV["SCOUT_WORKFLOW_DIR"] || 
-        begin 
+      @workflow_dir ||
+        ENV["SCOUT_WORKFLOW_DIR"] ||
+        begin
           workflow_dir_config = Path.setup("etc/workflow_dir")
           if workflow_dir_config.exists?
             Path.setup(workflow_dir_config.read.strip)
@@ -40,9 +40,9 @@ module Workflow
     end
 
     def workflow_repo
-      @workflow_repo || 
-        ENV["SCOUT_WORKFLOW_REPO"] || 
-        begin 
+      @workflow_repo ||
+        ENV["SCOUT_WORKFLOW_REPO"] ||
+        begin
           workflow_repo_config = Path.setup("etc/workflow_repo")
           if workflow_repo_config.exists?
             workflow_repo_config.read.strip

data/scout-gear.gemspec

@@ -2,11 +2,11 @@
 # DO NOT EDIT THIS FILE DIRECTLY
 # Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
-# stub: scout-gear 10.11.4 ruby lib
+# stub: scout-gear 10.11.7 ruby lib
 
 Gem::Specification.new do |s|
   s.name = "scout-gear".freeze
-  s.version = "10.11.4".freeze
+  s.version = "10.11.7".freeze
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
   s.require_paths = ["lib".freeze]
@@ -169,6 +169,7 @@ Gem::Specification.new do |s|
     "scout_commands/template",
     "scout_commands/update",
     "scout_commands/workflow/cmd",
+    "scout_commands/workflow/example",
     "scout_commands/workflow/info",
     "scout_commands/workflow/install",
     "scout_commands/workflow/list",
@@ -283,7 +284,7 @@ Gem::Specification.new do |s|
   ]
   s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
   s.licenses = ["MIT".freeze]
-  s.rubygems_version = "3.7.0.dev".freeze
+  s.rubygems_version = "3.7.2".freeze
   s.summary = "basic gear for scouts".freeze
 
   s.specification_version = 4

data/scout_commands/alias

@@ -17,7 +17,7 @@ earlier, like --dev or --log. Edit #{Scout.ect.cmd_alias.find} to fine tune.
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/batch/clean

@@ -25,7 +25,7 @@ $ #{$0} [<options>] <filename> [<other|->]*
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -109,7 +109,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  end
 
  if File.exist?(fstatus = File.join(dir, 'job.status'))
-   fstatus_txt = Open.read(fstatus) 
+   fstatus_txt = Open.read(fstatus)
    begin
      if job_batch_system == "lsf"
        nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
@@ -140,28 +140,28 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  aborted = error = true if ! done && aborted.nil? && error.nil?
  #if done || error || aborted || running || queued || jobid || search
  #  select = false
- #  select = true if done && exit_status && exit_status.to_i == 0 
- #  select = true if error && exit_status && exit_status.to_i != 0 
+ #  select = true if done && exit_status && exit_status.to_i == 0
+ #  select = true if error && exit_status && exit_status.to_i != 0
  #  select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
- #  select = select && jobid.split(",").include?(id) if jobid 
+ #  select = select && jobid.split(",").include?(id) if jobid
  #  select = select && cmd.match(/#{search}/) if search
- #  next unless select 
+ #  next unless select
  #end
 
- if done || error || aborted || queued || jobid 
+ if done || error || aborted || queued || jobid
    select = false
    select = true if done && exit_status == 0
-   select = true if error && exit_status && exit_status != 0 
+   select = true if error && exit_status && exit_status != 0
    select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
    is_running =  exit_status.nil? && ( (running_jobs.include?(id) && (!deps || (running_jobs & deps).empty?)) || different_system )
    select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
    select = true if jobid && jobid.split(",").include?(id)
    select = select && cmd.match(/#{search}/) if search
-   next unless select 
+   next unless select
  elsif search
    select = false
    select = true if search && cmd.match(/#{search}/)
-   next unless select 
+   next unless select
  end
 
 
@@ -180,7 +180,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
  if options[:batch_parameters]
    puts Log.color(:magenta, "BATCH parameters: ")
-   case job_batch_system 
+   case job_batch_system
    when 'slurm'
      puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
    when 'lsf'
@@ -206,6 +206,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
  Open.rm_rf dir unless dry_run
 end
 
-puts 
+puts
 puts "Found #{count} jobs"
 

data/scout_commands/batch/list

@@ -12,7 +12,7 @@ List all batch jobs
 
 $ #{$0} [<options>] <filename> [<other|->]*
 
-$ rbbt slurm list [options] 
+$ rbbt slurm list [options]
 
 -h--help Print this help
 -d--done Done jobs only
@@ -33,7 +33,7 @@ $ rbbt slurm list [options]
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -132,7 +132,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   end
 
   if File.exist?(fstatus = File.join(dir, 'job.status'))
-    fstatus_txt = Open.read(fstatus) 
+    fstatus_txt = Open.read(fstatus)
     begin
       if job_batch_system == "lsf"
         nodes = Open.read(fstatus).split("\n").last.split(/\s+/)[5].split(",")
@@ -170,47 +170,47 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
   cadeps = Open.read(fcadep).split("\n") if File.exist?(fcadep)
 
   is_running =  exit_status.nil? && ( (running_jobs.include?(id) && (deps.nil? || (running_jobs & deps).empty?)) || different_system )
-  if done || error || aborted || running || queued || jobid 
+  if done || error || aborted || running || queued || jobid
     select = false
     select = true if done && exit_status == 0
-    select = true if error && exit_status && exit_status != 0 
+    select = true if error && exit_status && exit_status != 0
     select = true if aborted && (exit_status.nil? && ! running_jobs.include?(id))
     select = true if queued && deps && (running_jobs & deps).any? || queued && is_running && nodes.empty?
     select = true if running && nodes.any? && (exit_status.nil? && running_jobs.include?(id)) && (!deps || (running_jobs & deps).empty?)
     select = true if jobid && jobid.split(",").include?(id)
     select = select && step_path.match(/#{search}/) if search
-    next unless select 
+    next unless select
   elsif search
     select = false
     select = true if search && cmd.match(/#{search}/)
-    next unless select 
+    next unless select
   end
 
 
   count += 1
 
   if options[:compressed] && tail.nil?
-    status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  Log.color(:green, id)  
+    status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) :  Log.color(:green, id)
     if different_system
-      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  
+      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)
     else
       #status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
-      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : 
-        (running_jobs.include?(id) || $norunningjobs ? 
-         (is_running ? Log.color(:cyan, id) : Log.color(:yellow, id) ) : 
+      status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :
+        (running_jobs.include?(id) || $norunningjobs ?
+         (is_running ? Log.color(:cyan, id) : Log.color(:yellow, id) ) :
          Log.color(:red, id) )
-    end 
+    end
     prog_rep = []
-    if options[:progress] 
+    if options[:progress]
       step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
       if step_line
         step_path = step_line.split(": ").last.strip
         step = Step.new step_path
         has_bar = false
         [step].reverse.each do |j|
-          next if j.done? 
+          next if j.done?
           if j.file(:progress).exists?
-            bar = Log::ProgressBar.new 
+            bar = Log::ProgressBar.new
             bar.load(j.file(:progress).yaml)
             rep = bar.report_msg.split("·")[1]
             rep = rep.sub(/.*?(\d+%)/, Log.color(:blue,'\1')).sub(/\-.*/,'')
@@ -238,7 +238,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
     puts Log.color(:magenta, "Job ID (#{Log.color(:red, job_batch_system)}): ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  Log.color(:green, id)  )
   else
     puts Log.color(:magenta, "Job ID: ") << (exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" :  (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) ))
-  end 
+  end
   puts Log.color(:magenta, "Dependencies: ") << deps * ", " if deps
   puts Log.color(:magenta, "Dependencies (can fail): ") << cadeps * ", " if cadeps
   puts Log.color(:magenta, "Nodes: ") << nodes * ", " if long
@@ -247,7 +247,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
   if options[:batch_parameters]
     puts Log.color(:magenta, "BATCH parameters: ")
-    case job_batch_system 
+    case job_batch_system
     when 'slurm'
       text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
     when 'lsf'
@@ -295,14 +295,14 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
     ticks = 1 if ticks == 0
     time_elapsed = 1 if time_elapsed == 0
     puts Log.color(:yellow, "CPU average: ") + "%.2f" % ( ticks / clock_ticks / time_elapsed * 100).to_s
-    puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s 
+    puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s
     puts Log.color(:yellow, "Time: ") + Misc.format_seconds((eend - start))
 
   end
 
   if options[:sacct_peformance]
     begin
-      raise "sacct not supported for LSF" unless batch_system == 'slurm' 
+      raise "sacct not supported for LSF" unless batch_system == 'slurm'
       tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
       values = tsv[tsv.keys.first]
       if values.compact.any?
@@ -324,20 +324,21 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
       puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
     else
       puts Log.color(:magenta, "Log tail: ")
-      puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | grep -v  \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
+      pad_tail = 1000 + tail.to_i
+      puts CMD.cmd("tail -n #{pad_tail} #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m'  | grep -v -e '^[[:space:]]*$' | grep -v  \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
     end
   end
 
-  if options[:progress] 
+  if options[:progress]
     step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
     if step_line
       step_path = step_line.split(": ").last.strip
       step = Step.new step_path
       has_bar = false
       (step.rec_dependencies.to_a + [step]).reverse.each do |j|
-        next if j.done? 
+        next if j.done?
         if j.file(:progress).exists?
-          bar = Log::ProgressBar.new 
+          bar = Log::ProgressBar.new
           bar.load(j.file(:progress).yaml)
           puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
           has_bar = true
@@ -352,6 +353,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
 
 end
 
-puts 
+puts
 puts Log.color :clear, "Found #{count} jobs"
 

data/scout_commands/batch/tail

@@ -8,14 +8,14 @@ options = SOPT.setup <<EOF
 
 Queue a job in Marenostrum
 
-$ rbbt slurm tail <directory|jobid> [options] 
+$ rbbt slurm tail <directory|jobid|step> [options]
 
 -h--help Print this help
 EOF
 
 if options[:help]
  if defined? rbbt_usage
-  rbbt_usage 
+  rbbt_usage
  else
   puts SOPT.doc
  end
@@ -29,18 +29,22 @@ directory = ARGV.shift
 
 raise ParameterException if directory.nil?
 
+workdir = File.expand_path('~/scout-batch')
+Path.setup(workdir)
+
 if directory =~ /^[0-9]*$/
-  workdir = File.expand_path('~/scout-batch')
-  Path.setup(workdir)
 
   workdir.glob("**/job.id").each do |file|
     next unless directory == Open.read(file).strip
     directory = File.dirname(file)
     break
   end
+elsif ! File.exist?(File.join(directory, 'command.batch'))
+  out = CMD.cmd("grep -r -l 'STEP_PATH: .*#{directory}' '#{workdir}'").read.split("\n").first
+  directory = out.sub("/command.batch", '') if out and not out.empty?
 end
 
-raise ParameterException, "Could not identify job #{directory}" unless File.exist?(directory)
+raise ParameterException, "Could not identify job #{directory}" unless File.exist?(File.join(directory, 'command.batch'))
 
 require 'rbbt/hpc/slurm'
 

data/scout_commands/cat

@@ -17,7 +17,7 @@ $ #{$0} [<options>] (<resource> <path>|<path>)
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/doc

@@ -16,7 +16,7 @@ If you don't specify the module it prints those available
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -26,7 +26,7 @@ end
 
 module_name = ARGV.first
 if module_name.nil?
-  puts Scout.doc.lib.scout.glob("**/*.md").collect{|f| 
+  puts Scout.doc.lib.scout.glob("**/*.md").collect{|f|
     f.find.unset_extension.sub(Scout.doc.lib.scout.find + "/", '')
   } * "\n"
 else

data/scout_commands/entity

@@ -21,7 +21,7 @@ type, property, entity = ARGV
 
 if options[:help] and type.nil?
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -63,12 +63,12 @@ end
 entity_module = Kernel.const_get type
 
 entity_module.annotations.uniq.each do |attr|
-  SOPT.register attr.slice(0,1).to_s, attr, true, "Entity attribute" 
+  SOPT.register attr.slice(0,1).to_s, attr, true, "Entity attribute"
 end
 
 if options[:help] and property.nil?
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -99,7 +99,7 @@ if options[:help]
   end * " "
 
   SOPT.synopsys['[<property_parameters>]*'] = property_str
-  
+
   if defined? scout_usage
     scout_usage
   else

data/scout_commands/find

@@ -17,7 +17,7 @@ $ #{$0} [<options>] (<resource> <path>|<path>)
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/kb/config

@@ -18,7 +18,7 @@ $ #{$0} [<options>] <name> [knowledge_base]
 EOF
 if options.delete :help
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/kb/entities

@@ -16,7 +16,7 @@ $ #{$0} [<options>] <entity> <identifier_files>
 EOF
 if options.delete :help
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/kb/list

@@ -16,7 +16,7 @@ $ #{$0} [<options>] <filename> [<other|->]*
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end