scout-gear 10.11.6 → 10.11.7

data/scout_commands/workflow/task

@@ -89,7 +89,7 @@ job = workflow.job(task_name, jobname, job_inputs)
 job.recursive_clean if recursive_clean
 job.clean if clean
 
-if clean_task 
+if clean_task
   ENV["SCOUT_UPDATE"] = 'true'
   clean_task.split(",").each do |clean_task|
     if clean_task.include? "#"
@@ -99,12 +99,13 @@ if clean_task
     job.rec_dependencies.each do |dep|
       next unless dep.task_name.to_s == clean_task.to_s
       next unless clean_workflow.nil? || clean_workflow == dep.workflow.to_s
-      dep.clean 
+      dep.clean
       dep.set_info :status, :cleaned
     end
 
     job.clean if (job.task_name.to_s == clean_task.to_s) || ! job.updated?
   end
+
 end
 
 if provenance
@@ -123,6 +124,8 @@ elsif save_inputs
 else
   require 'scout/workflow/deployment'
 
+  Workflow::Orchestrator.prepare_for_execution(job)
+
   if deploy_rules.nil?
     rules = Workflow::Orchestrator.load_rules_for_job(job)
   else
@@ -138,32 +141,54 @@ else
       puts save_inputs
       exit
     end
-  when 'dry', 'dryrun', 'debug'
+  when 'very_dry', 'vdry'
     batches = Workflow::Orchestrator.job_batches rules, job
+    batches = Workflow::Orchestrator.clean_batches batches
+    batches = Workflow::Orchestrator.sort_batches batches
+    batches.each do |batch|
+      top = batch[:top_level]
+      puts top.short_path
+    end
+    puts
+    puts "#{batches.length} batches"
+    exit 0
+  when 'dry', 'dryrun', 'dry_run', 'debug'
+    batches = Workflow::Orchestrator.job_batches rules, job
+    batches = Workflow::Orchestrator.sort_batches batches
     batches.each do |batch|
       top = batch[:top_level]
       jobs = batch[:jobs]
-      puts Log.color :title, Log.fingerprint(top)
-      puts batch[:rules].to_yaml
       deps = batch[:deps].collect{|dep| dep[:top_level] }
+      other = jobs - [top]
+      overriden = top.overriden?
       provided_inputs = jobs.inject({}){|acc,j| acc = IndiferentHash.add_defaults acc, j.provided_inputs }
+
+      puts Log.color :title, top.short_path
+      puts Log.color :subtitle, 'Other jobs: ' + Log.color(:blue, other.collect{|d| d.short_path}.uniq * ', ' ) if other.any?
+      puts Log.color :subtitle, 'Dependencies: ' + Log.color(:blue, deps.collect{|d| d.short_path}.uniq * ', ' ) if deps.any?
+      puts Log.color :subtitle, 'Overriden: ' + 'true' if overriden
+      batch[:rules].each do |k,v|
+        puts Log.color(:cyan, k + ": ") + v.to_json
+      end
       provided_inputs.each do |i,v|
         puts Log.color :green, "#{i}: " + Log.color(:blue, Log.fingerprint(v))
       end
-      other = jobs - [top]
-      puts Log.color :subtitle, 'Other jobs: ' + Log.color(:blue, other.collect{|d| d.short_path}.uniq * ', ' ) if other.any?
-      puts Log.color :subtitle, 'Dependencies: ' + Log.color(:blue, deps.collect{|d| d.short_path}.uniq * ', ' ) if deps.any?
     end
+    puts
+    puts "#{batches.length} batches"
     exit 0
   when "serial"
-    job.run(true) 
-  when "local"
+    job.run(true)
+  when "local", "workers"
     Workflow::LocalExecutor.produce job, rules
   when 'batch', 'sched', 'cluster', "slurm", 'pbs', 'lsf'
     Workflow::Scheduler.produce job, rules
     exit 0
+  when 'single_batch', 'single'
+    Workflow::Scheduler.produce_single_batch job, rules
+    exit 0
   else
-    raise 'Use scout-camp instead of scout' unless defined? OffsiteStep
+    raise "Deploy '#{deploy}' not standard, assumed to be a server name; use scout-camp instead of scout" unless defined? OffsiteStep
     if deploy.end_with?('-batch')
       server = deploy.sub('-batch','')
       OffsiteStep.setup(job, server: server, batch: true)

data/scout_commands/workflow/trace

@@ -21,7 +21,7 @@ $ #{$0} [<options>] <job-result>
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/scout_commands/workflow/write_info

@@ -16,7 +16,7 @@ $ #{$0} <job-result> <key> <value>
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end
@@ -63,7 +63,7 @@ if recursive && step.info[:archived_info]
       info[key] = value if (force || ! info.include?(key)) && (!check_pid || info[:pid].to_s == pid and info[:pid_hostname] == host)
     rescue
       Log.warn "Could no set info #{key} for archived_dep #{info[:path]}: #{$!.message}"
-    end 
+    end
   end
   step.set_info :archived_info, ad
 end

data/share/templates/command

@@ -14,7 +14,7 @@ $ #{$0} [<options>] <filename> [<other|->]*
 EOF
 if options[:help]
   if defined? scout_usage
-    scout_usage 
+    scout_usage
   else
     puts SOPT.doc
   end

data/test/scout/association/test_item.rb

@@ -9,6 +9,11 @@ class TestAssociationItem < Test::Unit::TestCase
     assert TSV === AssociationItem.incidence(pairs)
     assert_equal 2, AssociationItem.incidence(pairs).length
     assert_equal 2, AssociationItem.incidence(pairs).fields.length
+
+    associations = AssociationItem.setup(pairs)
+    associations.extend AnnotatedArray
+
+    assert_equal 2, associations.incidence.fields.length
   end
 
   def test_brothers

data/test/scout/entity/test_property.rb

@@ -70,7 +70,7 @@ class TestEntityProperty < Test::Unit::TestCase
     end
 
     property :annotation_list => :single do
-      self.chars.to_a.collect{|c| 
+      self.chars.to_a.collect{|c|
         ReversableString.setup(c)
       }
     end
@@ -80,11 +80,11 @@ class TestEntityProperty < Test::Unit::TestCase
     end
 
     $processed_multiple = []
-    property :multiple_annotation_list => :multiple do 
+    property :multiple_annotation_list => :multiple do
       $processed_multiple.concat self
       res = {}
       self.collect do |e|
-        e.chars.to_a.collect{|c| 
+        e.chars.to_a.collect{|c|
           ReversableString.setup(c)
         }
       end

data/test/scout/knowledge_base/test_description.rb

@@ -46,7 +46,7 @@ Sibling relationships.
       assert_include kb.description(:brothers), "Sibling relationships."
     end
   end
- 
+
   def test_full_description
     TmpFile.with_dir do |dir|
       brothers = datafile_test(:person).brothers

data/test/scout/knowledge_base/test_traverse.rb

@@ -126,8 +126,8 @@ end
 #
 #        kb.register :gene_ages, datadir_test.gene_ages, :source => "=>Associated Gene Name"
 #
-#        kb.register :CollecTRI, datadir_test.CollecTRI, 
-#          :source => "Transcription Factor=~Associated Gene Name", 
+#        kb.register :CollecTRI, datadir_test.CollecTRI,
+#          :source => "Transcription Factor=~Associated Gene Name",
 #          :target => "Target Gene=~Associated Gene Name",
 #          :fields => ["[ExTRI] Confidence", "[ExTRI] PMID"]
 #

data/test/scout/persist/engine/test_packed_index.rb

@@ -14,8 +14,8 @@ class TestClass < Test::Unit::TestCase
       pi.close
       pi = PackedIndex.new(tmpfile, false)
       100.times do |i|
-        assert_equal i, pi[i][0] 
-        assert_equal i+2, pi[i][1] 
+        assert_equal i, pi[i][0]
+        assert_equal i+2, pi[i][1]
       end
       assert_equal nil, pi[100]
       assert_equal nil, pi[101]
@@ -75,8 +75,8 @@ class TestClass < Test::Unit::TestCase
       pi = PackedIndex.new(tmpfile, false)
       Misc.benchmark(1000) do
         100.times do |i|
-          assert_equal i, pi[i][0] 
-          assert_equal i+2, pi[i][1] 
+          assert_equal i, pi[i][0]
+          assert_equal i+2, pi[i][1]
         end
       end
 
@@ -87,8 +87,8 @@ class TestClass < Test::Unit::TestCase
         end
         Misc.benchmark(1000) do
           100.times do |i|
-            assert_equal i, tk[i][0] 
-            assert_equal i+2, tk[i][1] 
+            assert_equal i, tk[i][0]
+            assert_equal i+2, tk[i][1]
           end
         end
       end

data/test/scout/persist/test_tsv.rb

@@ -14,7 +14,7 @@ row2    a    a    id3
     EOF
 
 
-    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+    tsv = Persist.persist("TEST Persist TSV", :tsv) do
       TmpFile.with_file(content) do |filename|
         TSV.open(filename)
       end
@@ -25,7 +25,7 @@ row2    a    a    id3
     assert_include tsv.keys, 'row1'
     assert_include tsv.keys, 'row2'
 
-    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+    tsv = Persist.persist("TEST Persist TSV", :tsv) do
       TmpFile.with_file(content) do |filename|
         TSV.open(filename)
       end
@@ -35,7 +35,7 @@ row2    a    a    id3
     assert_include tsv.keys, 'row2'
 
     assert_nothing_raised do
-      tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+      tsv = Persist.persist("TEST Persist TSV", :tsv) do
         raise
       end
     end
@@ -76,7 +76,7 @@ row2    a    a    id3
 
     tsv.close
     assert_nothing_raised do
-      tsv = Persist.persist("TEST Persist TSV", :HDB) do 
+      tsv = Persist.persist("TEST Persist TSV", :HDB) do
         raise
       end
     end

data/test/scout/persist/tsv/adapter/test_packed_index.rb

@@ -16,16 +16,16 @@ class TestClass < Test::Unit::TestCase
 
       pi = PackedIndex.new(tmpfile, false)
       100.times do |i|
-        assert_equal i, pi[i][0] 
-        assert_equal i+2, pi[i][1] 
+        assert_equal i, pi[i][0]
+        assert_equal i+2, pi[i][1]
       end
       assert_equal nil, pi[100]
       assert_equal nil, pi[101]
 
       pi = Persist.open_pki tmpfile, false, %w(i i 23s f f f f f)
       100.times do |i|
-        assert_equal i, pi[i][0] 
-        assert_equal i+2, pi[i][1] 
+        assert_equal i, pi[i][0]
+        assert_equal i+2, pi[i][1]
       end
       assert_equal nil, pi[100]
       assert_equal nil, pi[101]

data/test/scout/persist/tsv/adapter/test_sharder.rb

@@ -11,7 +11,7 @@ class TestSharder < Test::Unit::TestCase
 
       size = 10
       sharder = Persist.open_sharder(dir, true, :HDB, :path => dir, :shard_function => shard_function, :persist => true, :serializer => :float_array)
-      size.times do |v| 
+      size.times do |v|
         sharder[v.to_s] = [v, v*2]
       end
       assert_equal dir, sharder.persistence_path
@@ -97,15 +97,15 @@ id11,a11,b11
       size = 10
       sharder = Persist.tsv(dir,
                             persist_options: {
-                              :engine => 'fwt', 
-                              :path => dir, 
+                              :engine => 'fwt',
+                              :path => dir,
                               :serializer => :float,
-                              :range => false, :value_size => 64, 
-                              :shard_function => shard_function, 
+                              :range => false, :value_size => 64,
+                              :shard_function => shard_function,
                               :pos_function => pos_function
                             }
                            ) do |db|
-                             size.times do |v| 
+                             size.times do |v|
                                v = v + 1
                                chr = "chr" << (v % 5).to_s
                                key = chr + ":" << v.to_s
@@ -131,7 +131,7 @@ id11,a11,b11
 
       assert_equal 4.0, sharder["chr2:2"]
 
-      assert_equal size, sharder.size 
+      assert_equal size, sharder.size
     end
   end
 
@@ -147,19 +147,19 @@ id11,a11,b11
 
       size = 10
       chrs = (1..10).to_a
-      sharder = Persist.tsv(dir, 
-                            engine: 'pki', 
-                            :persist_options => { 
+      sharder = Persist.tsv(dir,
+                            engine: 'pki',
+                            :persist_options => {
                               :pattern =>  %w(f f),
-                              :range => false, 
-                              :value_size => 64, 
-                              :file => dir, 
+                              :range => false,
+                              :value_size => 64,
+                              :file => dir,
                               :shard_function => shard_function,
                               :pos_function => pos_function
                             }) do |db|
         chrs.each do |c|
-          size.times do |v| 
-            v = v 
+          size.times do |v|
+            v = v
             chr = "chr" << c.to_s
             key = chr + ":" << v.to_s
             db[key] = [v, v*2]
@@ -184,7 +184,7 @@ id11,a11,b11
       assert_equal count, size*chrs.length
 
       sharder = Persist.open_sharder(
-        dir, false, 'pki', 
+        dir, false, 'pki',
         {:pattern => %w(f f), :file => dir, :range => false, :value_size => 64, :pos_function => pos_function}, &shard_function
       )
 
@@ -209,7 +209,7 @@ id11,a11,b11
       chrs = (1..10).to_a
       sharder = Persist.tsv(dir, persist_options: {:pattern => %w(f), :range => false, :value_size => 64, :engine => 'pki', :file => dir, :shard_function => shard_function, :pos_function => pos_function}) do |db|
         chrs.each do |c|
-          size.times do |v| 
+          size.times do |v|
             v = v + 1
             chr = "chr" << c.to_s
             key = chr + ":" << (v*2).to_s
@@ -249,15 +249,15 @@ id11,a11,b11
 
       size = 10
       sharder = Persist.persist_tsv("ShardTSV_FWT", nil, {}, {
-                              :engine => 'fwt', 
-                              :path => dir, 
+                              :engine => 'fwt',
+                              :path => dir,
                               :serializer => :float,
-                              :range => false, :value_size => 64, 
-                              :shard_function => shard_function, 
+                              :range => false, :value_size => 64,
+                              :shard_function => shard_function,
                               :pos_function => pos_function
                             }
                            ) do |db|
-                             size.times do |v| 
+                             size.times do |v|
                                v = v + 1
                                chr = "chr" << (v % 5).to_s
                                key = chr + ":" << v.to_s
@@ -282,7 +282,7 @@ id11,a11,b11
 
       assert_equal 4.0, sharder["chr2:2"]
 
-      assert_equal size, sharder.size 
+      assert_equal size, sharder.size
     end
   end
 

data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb

@@ -112,7 +112,7 @@ row2    0.1  4.5 0
       assert_equal [0.2, 0.3, 0], tsv2["row1"]
       assert_equal TSVAdapter::FloatArraySerializer, tsv2.serializer
     end
- 
+
   end
 
   def test_float_double

data/test/scout/persist/tsv/test_serialize.rb

@@ -6,7 +6,7 @@ class TestClass < Test::Unit::TestCase
     m = TSVAdapter.serializer_module :marshal
     v = [1, :a]
     d = m.dump(v)
-    assert_equal v, m.load(d) 
+    assert_equal v, m.load(d)
   end
 end
 

data/test/scout/test_association.rb

@@ -40,7 +40,7 @@ class TestAssociation < Test::Unit::TestCase
     assert_equal "2021", database["Clei"]["Date"]
     assert_include database.key_field, "Alias"
   end
-  
+
   def test_brothers_id
     database = Association.database(datadir_test.person.brothers, :source => "Older=~Older (Alias)=>Name", :target => "Younger=~Younger (Alias)=>ID")
     assert_equal '001', database["Isabel"]["Younger"]

data/test/scout/test_tsv.rb

@@ -27,7 +27,7 @@ row4    a    a    id3
     tsv = TmpFile.with_file(content) do |filename|
       TSV.open(filename)
     end
-    
+
     TmpFile.with_file(content2) do |filename|
       TSV.open(filename, :data => tsv)
     end
@@ -228,7 +228,7 @@ row2    A AA AAA
 
   def test_flat_key
     content =<<-EOF
-#Id    ValueA 
+#Id    ValueA
 row1   a   aa   aaa
 row2   b  bbb bbbb bb aa
     EOF

data/test/scout/test_workflow.rb

@@ -42,7 +42,7 @@ class TestWorkflow < Test::Unit::TestCase
     end
 
     dep :prepare_batter
-    task :bake_muffin_tray => :string do 
+    task :bake_muffin_tray => :string do
       bake(step(:prepare_batter).load)
     end
 
@@ -53,7 +53,7 @@ class TestWorkflow < Test::Unit::TestCase
       batter = mix(whisked_eggs, Pantry.flour.produce)
 
       if add_bluberries
-        batter = mix(batter, Pantry.blueberries.produce) 
+        batter = mix(batter, Pantry.blueberries.produce)
       end
 
       batter

data/test/scout/tsv/test_annotation.rb

@@ -15,10 +15,10 @@ class TestAnnotationTSV < Test::Unit::TestCase
     AnnotationClass.setup(str1, :c11, :c12)
     AnnotationClass.setup(str2, :c21, :c22)
 
-    assert_equal str1, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str1.annotation_id + "#0"]["literal"] 
-    assert_equal :c11, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str1.annotation_id + "#0"]["code"] 
-    assert_equal str2, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str2.annotation_id + "#1"]["literal"] 
-    assert_equal :c21, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str2.annotation_id + "#1"]["code"] 
+    assert_equal str1, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str1.annotation_id + "#0"]["literal"]
+    assert_equal :c11, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str1.annotation_id + "#0"]["code"]
+    assert_equal str2, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str2.annotation_id + "#1"]["literal"]
+    assert_equal :c21, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str2.annotation_id + "#1"]["code"]
     assert_equal "c11", JSON.parse(Annotation.tsv([str1, str2], :code, :JSON).tap{|t| t.unnamed = false}[str1.annotation_id + "#0"]["JSON"])["code"]
   end
 

data/test/scout/tsv/test_index.rb

@@ -15,7 +15,7 @@ class TestTSVIndex < Test::Unit::TestCase
     end
 
     tsv = tsv.slice ["Start", "End"]
- 
+
     tsv
   end
 

data/test/scout/tsv/test_open.rb

@@ -78,7 +78,7 @@ class TestOpenTraverse < Test::Unit::TestCase
     num_lines = 100
     lines = num_lines.times.collect{|i| "line-#{i}" }
 
-    assert_raise ScoutException do 
+    assert_raise ScoutException do
       i = 0
       Log.with_severity 7 do
         dumper = TSV::Dumper.new :key_field => "Key", :fields => ["Value"], :type => :single
@@ -125,7 +125,7 @@ row2 aa bb cc
 
     s = StringIO.new text
     collapsed = TSV.collapse_stream(s)
-    tsv = TSV.open collapsed 
+    tsv = TSV.open collapsed
     assert_equal ["A", "a"], tsv["row1"][0]
     assert_equal ["BB", "bb"], tsv["row2"][1]
   end

data/test/scout/tsv/test_parser.rb

@@ -15,7 +15,7 @@ class TestTSVParser < Test::Unit::TestCase
   def test_parse_line_key
     line = (0..10).to_a * "\t"
     key, values = TSV.parse_line(line, key: 2)
-    
+
     assert_equal "2", key
     assert_equal %w(0 1 3 4 5 6 7 8 9 10), values
   end
@@ -33,7 +33,7 @@ class TestTSVParser < Test::Unit::TestCase
     num = 10_000
     txt = num.times.inject(nil) do |acc,i|
       (acc.nil? ? "" : acc << "\n") << (0..10).collect{|v| v == 0 ? i : [v,v] * "|" } * "\t"
-    end 
+    end
 
     txt = StringIO.new(([txt] * (10))*"\n")
     Misc.benchmark 5 do

data/test/scout/tsv/test_stream.rb

@@ -14,7 +14,7 @@ row3 AAA BBB CCC
 
     text2=<<-EOF
 #: :sep=" "
-#Row Labela Labelb 
+#Row Labela Labelb
 row1 a b
 row2 aa bb
 row3 aaa bbb

data/test/scout/tsv/test_transformer.rb

@@ -26,7 +26,7 @@ row2    A2|A22    B2|B22
 
       assert_equal %w(A1 A11 B1 B11), tsv['row1']
     end
-    
+
   end
 
   def test_traverse

data/test/scout/tsv/util/test_filter.rb

@@ -182,7 +182,7 @@ row2    A    B
       tsv.pop_filter
       assert tsv.filename !~ /ValueA/
     end
- 
+
   end
 end
 

data/test/scout/tsv/util/test_melt.rb

@@ -11,7 +11,7 @@ class TestClass < Test::Unit::TestCase
 10,20,30
 20,40,60
     EOF
-    
+
     target =<<-EOF
 #: :sep=,#:type=:list
 #ID,Size,Time,System

data/test/scout/tsv/util/test_reorder.rb

@@ -83,7 +83,7 @@ row1    a    b    Id1
 row2    A    B    Id3
 row3    a    C    Id4
     EOF
- 
+
     TmpFile.with_file(content) do |filename|
       tsv = TSV.open(filename, :sep => /\s+/)
 

data/test/scout/work_queue/test_socket.rb

@@ -6,7 +6,7 @@ class TestSocket < Test::Unit::TestCase
   class DoneProcessing end
 
   def test_simple
-    socket = WorkQueue::Socket.new 
+    socket = WorkQueue::Socket.new
 
     socket.write 1
     socket.write 2
@@ -25,7 +25,7 @@ class TestSocket < Test::Unit::TestCase
   end
 
   def __test_speed
-    socket = WorkQueue::Socket.new 
+    socket = WorkQueue::Socket.new
 
     num = 50_000
 
@@ -33,7 +33,7 @@ class TestSocket < Test::Unit::TestCase
       num.times do |i|
         socket.write nil
       end
-      socket.write DoneProcessing.new 
+      socket.write DoneProcessing.new
     end
 
     bar = Log::ProgressBar.new num

data/test/scout/work_queue/test_worker.rb

@@ -46,7 +46,7 @@ class TestQueueWorker < Test::Unit::TestCase
           Open.consume_stream(sout, false, outfile)
           txt = Open.read(outfile)
           pid_list = txt.split("\n")
-          
+
           assert_equal (num_lines + 2) * num_workers, pid_list.length
 
           assert_nothing_raised do
@@ -122,7 +122,7 @@ class TestQueueWorker < Test::Unit::TestCase
       end
     end
 
-    read = Thread.new do 
+    read = Thread.new do
       begin
         while obj = output.read
           if DoneProcessing === obj

data/test/scout/workflow/deployment/orchestrator/test_batches.rb

@@ -35,7 +35,7 @@ module TestWFC
   dep :c1
   dep :c2
   task :c3 => :string do self.task_name.to_s end
-  
+
   dep_task :c4, TestWFC, :c3
 end
 
@@ -99,15 +99,25 @@ TestWFC:
     skip: true
     log: 4
     time: 10s
+  c4:
+    skip: true
   EOF
 
   def test_chain_batches
     job = TestWFD.job(:d1, nil)
     job.recursive_clean
 
-    job_chains_map = Workflow::Orchestrator.job_chains(RULES, job)
+    job_chain_list = []
+
+    job_chains = Workflow::Orchestrator.job_chains(RULES, job)
+    job_chains.each do |chain,list|
+      list.each do |info|
+        job_chain_list << [chain,info]
+      end
+    end
+
     workload = Workflow::Orchestrator.job_workload(job)
-    batches = Workflow::Orchestrator.chain_batches(RULES, job_chains_map, workload)
+    batches = Workflow::Orchestrator.chain_batches(RULES, job_chain_list, workload)
     assert_equal 3, batches.length
   end