scout-gear 10.11.4 → 10.11.7

data/scout_commands/kb/query

@@ -21,7 +21,7 @@ $ #{$0} [<options>] <name> <entity>
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -70,7 +70,7 @@ if matches.any?
   else
     matches.each do |match|
       puts Log.color :magenta, (reverse ? match.invert : match)
-      puts index[match].prety_print 
+      puts index[match].prety_print
     end
   end
 else

data/scout_commands/kb/register

@@ -23,7 +23,7 @@ $ #{$0} [<options>] <name> <filename>
 EOF
 if options.delete :help
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/kb/show

@@ -16,7 +16,7 @@ $ #{$0} [<options>] <name>
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/kb/traverse

@@ -20,7 +20,7 @@ $ #{$0} [<options>] <traversal>
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/log

@@ -23,7 +23,7 @@ NONE
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -31,11 +31,11 @@ if options[:help]
 end
 
 if ARGV.empty?
-  if Scout.etc.log_severity.exists?                                                                                                                                                         
-    puts Scout.etc.log_severity.read                                                                                                                                                      
-  else                                                                                                                                                                                     
-    puts Scout.etc.log_severity.find + ' does not exist'                                                                                                                                  
-  end                                                                                                                                                                                      
+  if Scout.etc.log_severity.exists?
+    puts Scout.etc.log_severity.read
+  else
+    puts Scout.etc.log_severity.find + ' does not exist'
+  end
 else
   level = ARGV[0]
   level = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE)[level.to_i] if level =~ /^\d+$/

data/scout_commands/resource/produce

@@ -8,7 +8,7 @@ options = SOPT.setup <<EOF
 
 Produce a resource
 
-$ #{$0} [<options>] <Resource> <file> 
+$ #{$0} [<options>] <Resource> <file>
 
 -h--help Print this help
 -W--workflows* Workflows to use; 'all' for all in Scout.etc.workflows:
@@ -17,7 +17,7 @@ $ #{$0} [<options>] <Resource> <file>
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/resource/sync

@@ -19,7 +19,7 @@ Path map defaults to :user
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/system/clean

@@ -47,7 +47,7 @@ if locks.any?
   locks.each do |file,info|
     if force or (info[:pid] && ! Misc.pid_alive?(info[:pid]))
       puts "  Removing #{ file }"
-      File.unlink file 
+      File.unlink file
     end
   end
 end
@@ -59,7 +59,7 @@ if persists.any?
   persists.each do |file,info|
     if force or (info[:pid] and Misc.pid_alive? info[:pid])
       puts "  Removing #{ file }"
-      File.unlink file 
+      File.unlink file
     end
   end
 end
@@ -71,7 +71,7 @@ if sensiblewrites.any?
   sensiblewrites.each do |file,info|
     if force or (info[:pid] and Misc.pid_alive? info[:pid])
       puts "  Removing #{ file }"
-      File.unlink file 
+      File.unlink file
     end
   end
 end
@@ -134,13 +134,13 @@ TSV.traverse jobs do |file,i|
     end
   end
 
-  if (force && status !~ /done/) or 
+  if (force && status !~ /done/) or
       status =~ /\b(old|dirty|nopid|error|missing|aborted|dead|sync|waiting)$/ or
       (status == "noinfo" and not done) or
-      status == "" 
+      status == ""
 
-    puts "  Removing #{ file } - #{status}" 
-    Step.clean(file) 
+    puts "  Removing #{ file } - #{status}"
+    Step.clean(file)
   end
 end
 

data/scout_commands/system/status

@@ -177,7 +177,7 @@ workflows.sort.each do |workflow,tasks|
       if options[:quick] and i[:done]
         status = 'done'
         str << " #{ Step.prov_status_msg status }"
-        if inputs and inputs.any? 
+        if inputs and inputs.any?
           str << input_msg(file, inputs)
         end
 
@@ -210,15 +210,15 @@ workflows.sort.each do |workflow,tasks|
         status = status.to_s
         if status != "done" and pid and not Misc.pid_alive?(pid)
           if File.exist? file
-            status << Log.color(:red, " (out of sync)") 
+            status << Log.color(:red, " (out of sync)")
           else
-            status << Log.color(:red, " (dead)") 
+            status << Log.color(:red, " (dead)")
           end
         end
         str << " #{ Step.prov_status_msg status }"
         str << " (dirty)" if status == 'done' && Step.new(file).dirty?
 
-        if inputs and inputs.any? 
+        if inputs and inputs.any?
           str << input_msg(file, inputs)
         end
 

data/scout_commands/template

@@ -19,7 +19,7 @@ Print a template making the substitutions that follow
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/update

@@ -16,7 +16,7 @@ If no gem is specified 'scout-gear' is selected.
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/workflow/cmd

@@ -2,6 +2,7 @@
 
 require 'scout'
 
+exec = $0
 $0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
 
 options = SOPT.setup <<EOF
@@ -76,7 +77,7 @@ begin
           CMD.cmd_log('ruby', wf.libdir["test_workflow.rb"].find)
         else
           Log.info "No bootstrap for #{ workflow }, running examples instead"
-          CMD.cmd_log("scout workflow example #{ workflow }")
+          CMD.cmd_log("#{exec} workflow example #{ workflow }")
           exit 0
         end
       end

data/scout_commands/workflow/example

@@ -0,0 +1,123 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'scout'
+
+cmd = $previous_commands ?
+  "scout #{$previous_commands.any? ? "#{$previous_commands * ' '} " : ''}#{File.basename(__FILE__)}" :
+  $PROGRAM_NAME
+
+options = SOPT.setup <<~EOF
+Run examples
+
+$ #{cmd} <workflow> [<task>] [<name>] -- [<task_options>]
+
+Run workflow examples. You can define tasks and example names. You can also override task options
+
+-h--help Print this help
+EOF
+if options[:help]
+  if defined? scout_usage
+    scout_usage
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+help = IndiferentHash.process_options options, :help
+
+filename, _ = ARGV
+
+
+raise MissingParameterException, :filename if filename.nil?
+
+def run_task(workflow, task, name)
+  Log.info "Running example #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
+
+  example_dir = workflow.libdir.examples[task][name].find
+  Log.debug "Using #{example_dir}"
+
+  ARGV.replace([workflow.to_s, task, '--load_inputs', example_dir, '--jobname', name,'-pf'] + $saved_args)
+
+  path = nil
+  success = nil
+  TmpFile.with_file do |res|
+    Open.open(res, :mode => 'w') do |file|
+      @pid = Process.fork{
+        STDOUT.reopen res
+        load Rbbt.share.rbbt_commands.workflow.task.find
+      }
+      Signal.trap(:INT) do
+        begin
+          Process.kill "INT", @pid
+        ensure
+          Kernel.exit! -1
+        end
+      end
+      file.close
+      begin
+        p,s = Process.waitpid2 @pid
+        success = s.success?
+      rescue Errno::ECHILD
+        success = true
+      end
+    end
+    sleep 0.5
+    path = Open.read(res).strip.split("\n").last if File.exist? res
+  end
+  path = "NO RESULT" if path.nil? or path.empty?
+
+  if success
+    Log.info "#{Log.color :green, "SUCCESS"} #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
+    return [path, true]
+  else
+    Log.info "#{Log.color :red, "ERROR"} #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
+    return [path, false]
+  end
+end
+
+workflow = ARGV.shift
+raise ParameterException if workflow.nil?
+
+task = ARGV.shift
+task = nil if task == '--'
+
+name = ARGV.shift if task
+name = nil if name == '--'
+
+orig_name = name
+
+$saved_args = ARGV.dup
+
+$saved_args.shift if $saved_args.include? '--'
+
+workflow = Workflow.require_workflow workflow
+
+tasks = task ? [task] : workflow.libdir.examples.glob('*').collect{|file| File.basename file }
+
+task_result = {}
+TSV.traverse tasks do |task|
+  names = name ? [name] : workflow.libdir.examples[task].glob('*').collect{|file| File.basename file }
+  TSV.traverse names  do |name|
+    success = run_task workflow, task, name
+    task_result[[task, name]] = success
+  end
+end
+
+
+task_result.each do |code,res|
+  task, name = code
+  path, success = res
+  if success
+    if orig_name
+      puts Open.read(path)
+    else
+      STDERR.puts "#{Log.color :green, "SUCCESS"} #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
+      puts path
+    end
+  else
+    STDERR.puts "#{Log.color :red, "ERROR"} #{Log.color :magenta, workflow.to_s}##{Log.color :yellow, task} -- #{Log.color :cyan, name}"
+    puts path
+  end
+end

data/scout_commands/workflow/info

@@ -16,7 +16,7 @@ $ #{$0} [<options>] <step_path>
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -38,6 +38,7 @@ elsif options[:recursive_inputs]
   end
 else
   step.info.each do |k,v|
+    next if k.to_s == 'exception'
     case v
     when nil
       next
@@ -50,5 +51,13 @@ else
   rescue
     next
   end
+  if step.error?
+    if exception = step.exception
+      begin
+        Log.exception exception
+      rescue
+      end
+    end
+  end
 end
 

data/scout_commands/workflow/install

@@ -22,7 +22,7 @@ workflows install automatically as required.
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/workflow/list

@@ -8,13 +8,13 @@ options = SOPT.setup <<EOF
 
 List all workflows
 
-$ #{$0} [<options>] 
+$ #{$0} [<options>]
 
 -h--help Print this help
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/workflow/process

@@ -22,7 +22,7 @@ $ #{$0} [<options>] ([<workflow>] [<task>] [<name>] | <filename>)
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -69,7 +69,7 @@ begin
   end
 
   jobs = files.collect{|file| Workflow.queue_job(file) }
-  
+
 
   begin
     Workflow.produce(jobs, **options)

data/scout_commands/workflow/prov

@@ -19,7 +19,7 @@ $ #{$0} [<options>] <filename> [<other|->]*
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -77,10 +77,10 @@ def adjacency(step)
     info[:dependencies].each do |task,name,path|
       dep = get_step path
       _id, _edges, _node_info = adjacency(dep)
-      edges << [_id, id] 
+      edges << [_id, id]
       edges.concat _edges
       node_info.merge!(_node_info)
-    end 
+    end
   end
 
   [id, edges, node_info]

data/scout_commands/workflow/task

@@ -89,7 +89,7 @@ job = workflow.job(task_name, jobname, job_inputs)
 job.recursive_clean if recursive_clean
 job.clean if clean
 
-if clean_task 
+if clean_task
   ENV["SCOUT_UPDATE"] = 'true'
   clean_task.split(",").each do |clean_task|
     if clean_task.include? "#"
@@ -99,12 +99,13 @@ if clean_task
     job.rec_dependencies.each do |dep|
       next unless dep.task_name.to_s == clean_task.to_s
       next unless clean_workflow.nil? || clean_workflow == dep.workflow.to_s
-      dep.clean 
+      dep.clean
       dep.set_info :status, :cleaned
     end
 
     job.clean if (job.task_name.to_s == clean_task.to_s) || ! job.updated?
   end
+
 end
 
 if provenance
@@ -123,6 +124,8 @@ elsif save_inputs
 else
   require 'scout/workflow/deployment'
 
+  Workflow::Orchestrator.prepare_for_execution(job)
+
   if deploy_rules.nil?
     rules = Workflow::Orchestrator.load_rules_for_job(job)
   else
@@ -138,32 +141,54 @@ else
       puts save_inputs
       exit
     end
-  when 'dry', 'dryrun', 'debug'
+  when 'very_dry', 'vdry'
     batches = Workflow::Orchestrator.job_batches rules, job
+    batches = Workflow::Orchestrator.clean_batches batches
+    batches = Workflow::Orchestrator.sort_batches batches
+    batches.each do |batch|
+      top = batch[:top_level]
+      puts top.short_path
+    end
+    puts
+    puts "#{batches.length} batches"
+    exit 0
+  when 'dry', 'dryrun', 'dry_run', 'debug'
+    batches = Workflow::Orchestrator.job_batches rules, job
+    batches = Workflow::Orchestrator.sort_batches batches
     batches.each do |batch|
       top = batch[:top_level]
       jobs = batch[:jobs]
-      puts Log.color :title, Log.fingerprint(top)
-      puts batch[:rules].to_yaml
       deps = batch[:deps].collect{|dep| dep[:top_level] }
+      other = jobs - [top]
+      overriden = top.overriden?
       provided_inputs = jobs.inject({}){|acc,j| acc = IndiferentHash.add_defaults acc, j.provided_inputs }
+
+      puts Log.color :title, top.short_path
+      puts Log.color :subtitle, 'Other jobs: ' + Log.color(:blue, other.collect{|d| d.short_path}.uniq * ', ' ) if other.any?
+      puts Log.color :subtitle, 'Dependencies: ' + Log.color(:blue, deps.collect{|d| d.short_path}.uniq * ', ' ) if deps.any?
+      puts Log.color :subtitle, 'Overriden: ' + 'true' if overriden
+      batch[:rules].each do |k,v|
+        puts Log.color(:cyan, k + ": ") + v.to_json
+      end
       provided_inputs.each do |i,v|
         puts Log.color :green, "#{i}: " + Log.color(:blue, Log.fingerprint(v))
       end
-      other = jobs - [top]
-      puts Log.color :subtitle, 'Other jobs: ' + Log.color(:blue, other.collect{|d| d.short_path}.uniq * ', ' ) if other.any?
-      puts Log.color :subtitle, 'Dependencies: ' + Log.color(:blue, deps.collect{|d| d.short_path}.uniq * ', ' ) if deps.any?
     end
+    puts
+    puts "#{batches.length} batches"
     exit 0
   when "serial"
-    job.run(true) 
-  when "local"
+    job.run(true)
+  when "local", "workers"
     Workflow::LocalExecutor.produce job, rules
   when 'batch', 'sched', 'cluster', "slurm", 'pbs', 'lsf'
     Workflow::Scheduler.produce job, rules
     exit 0
+  when 'single_batch', 'single'
+    Workflow::Scheduler.produce_single_batch job, rules
+    exit 0
   else
-    raise 'Use scout-camp instead of scout' unless defined? OffsiteStep
+    raise "Deploy '#{deploy}' not standard, assumed to be a server name; use scout-camp instead of scout" unless defined? OffsiteStep
     if deploy.end_with?('-batch')
       server = deploy.sub('-batch','')
       OffsiteStep.setup(job, server: server, batch: true)

data/scout_commands/workflow/trace

@@ -21,7 +21,7 @@ $ #{$0} [<options>] <job-result>
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/workflow/write_info

@@ -16,7 +16,7 @@ $ #{$0} <job-result> <key> <value>
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -63,7 +63,7 @@ if recursive && step.info[:archived_info]
       info[key] = value if (force || ! info.include?(key)) && (!check_pid || info[:pid].to_s == pid and info[:pid_hostname] == host)
     rescue
       Log.warn "Could no set info #{key} for archived_dep #{info[:path]}: #{$!.message}"
-    end 
+    end
   end
   step.set_info :archived_info, ad
 end

data/share/templates/command

@@ -14,7 +14,7 @@ $ #{$0} [<options>] <filename> [<other|->]*
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/test/scout/association/test_item.rb

@@ -9,6 +9,11 @@ class TestAssociationItem < Test::Unit::TestCase
     assert TSV === AssociationItem.incidence(pairs)
     assert_equal 2, AssociationItem.incidence(pairs).length
     assert_equal 2, AssociationItem.incidence(pairs).fields.length
+
+    associations = AssociationItem.setup(pairs)
+    associations.extend AnnotatedArray
+
+    assert_equal 2, associations.incidence.fields.length
   end
 
   def test_brothers

data/test/scout/entity/test_property.rb

@@ -70,7 +70,7 @@ class TestEntityProperty < Test::Unit::TestCase
     end
 
     property :annotation_list => :single do
-      self.chars.to_a.collect{|c| 
+      self.chars.to_a.collect{|c|
         ReversableString.setup(c)
       }
     end
@@ -80,11 +80,11 @@ class TestEntityProperty < Test::Unit::TestCase
     end
 
     $processed_multiple = []
-    property :multiple_annotation_list => :multiple do 
+    property :multiple_annotation_list => :multiple do
       $processed_multiple.concat self
       res = {}
       self.collect do |e|
-        e.chars.to_a.collect{|c| 
+        e.chars.to_a.collect{|c|
           ReversableString.setup(c)
         }
       end

data/test/scout/knowledge_base/test_description.rb

@@ -46,7 +46,7 @@ Sibling relationships.
       assert_include kb.description(:brothers), "Sibling relationships."
     end
   end
- 
+
   def test_full_description
     TmpFile.with_dir do |dir|
       brothers = datafile_test(:person).brothers

data/test/scout/knowledge_base/test_traverse.rb

@@ -126,8 +126,8 @@ end
 #
 #        kb.register :gene_ages, datadir_test.gene_ages, :source => "=>Associated Gene Name"
 #
-#        kb.register :CollecTRI, datadir_test.CollecTRI, 
-#          :source => "Transcription Factor=~Associated Gene Name", 
+#        kb.register :CollecTRI, datadir_test.CollecTRI,
+#          :source => "Transcription Factor=~Associated Gene Name",
 #          :target => "Target Gene=~Associated Gene Name",
 #          :fields => ["[ExTRI] Confidence", "[ExTRI] PMID"]
 #

data/test/scout/persist/engine/test_packed_index.rb

@@ -14,8 +14,8 @@ class TestClass < Test::Unit::TestCase
       pi.close
       pi = PackedIndex.new(tmpfile, false)
       100.times do |i|
-        assert_equal i, pi[i][0] 
-        assert_equal i+2, pi[i][1] 
+        assert_equal i, pi[i][0]
+        assert_equal i+2, pi[i][1]
       end
       assert_equal nil, pi[100]
       assert_equal nil, pi[101]
@@ -75,8 +75,8 @@ class TestClass < Test::Unit::TestCase
       pi = PackedIndex.new(tmpfile, false)
       Misc.benchmark(1000) do
         100.times do |i|
-          assert_equal i, pi[i][0] 
-          assert_equal i+2, pi[i][1] 
+          assert_equal i, pi[i][0]
+          assert_equal i+2, pi[i][1]
         end
       end
 
@@ -87,8 +87,8 @@ class TestClass < Test::Unit::TestCase
         end
         Misc.benchmark(1000) do
           100.times do |i|
-            assert_equal i, tk[i][0] 
-            assert_equal i+2, tk[i][1] 
+            assert_equal i, tk[i][0]
+            assert_equal i+2, tk[i][1]
           end
         end
       end