scout-gear 10.11.4 → 10.11.7

data/test/scout/persist/test_tsv.rb

@@ -14,7 +14,7 @@ row2    a    a    id3
     EOF
 
 
-    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+    tsv = Persist.persist("TEST Persist TSV", :tsv) do
       TmpFile.with_file(content) do |filename|
         TSV.open(filename)
       end
@@ -25,7 +25,7 @@ row2    a    a    id3
     assert_include tsv.keys, 'row1'
     assert_include tsv.keys, 'row2'
 
-    tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+    tsv = Persist.persist("TEST Persist TSV", :tsv) do
       TmpFile.with_file(content) do |filename|
         TSV.open(filename)
       end
@@ -35,7 +35,7 @@ row2    a    a    id3
     assert_include tsv.keys, 'row2'
 
     assert_nothing_raised do
-      tsv = Persist.persist("TEST Persist TSV", :tsv) do 
+      tsv = Persist.persist("TEST Persist TSV", :tsv) do
         raise
       end
     end
@@ -76,7 +76,7 @@ row2    a    a    id3
 
     tsv.close
     assert_nothing_raised do
-      tsv = Persist.persist("TEST Persist TSV", :HDB) do 
+      tsv = Persist.persist("TEST Persist TSV", :HDB) do
         raise
       end
     end

data/test/scout/persist/tsv/adapter/test_packed_index.rb

@@ -16,16 +16,16 @@ class TestClass < Test::Unit::TestCase
 
       pi = PackedIndex.new(tmpfile, false)
       100.times do |i|
-        assert_equal i, pi[i][0] 
-        assert_equal i+2, pi[i][1] 
+        assert_equal i, pi[i][0]
+        assert_equal i+2, pi[i][1]
       end
       assert_equal nil, pi[100]
       assert_equal nil, pi[101]
 
       pi = Persist.open_pki tmpfile, false, %w(i i 23s f f f f f)
       100.times do |i|
-        assert_equal i, pi[i][0] 
-        assert_equal i+2, pi[i][1] 
+        assert_equal i, pi[i][0]
+        assert_equal i+2, pi[i][1]
       end
       assert_equal nil, pi[100]
       assert_equal nil, pi[101]

data/test/scout/persist/tsv/adapter/test_sharder.rb

@@ -11,7 +11,7 @@ class TestSharder < Test::Unit::TestCase
 
       size = 10
       sharder = Persist.open_sharder(dir, true, :HDB, :path => dir, :shard_function => shard_function, :persist => true, :serializer => :float_array)
-      size.times do |v| 
+      size.times do |v|
         sharder[v.to_s] = [v, v*2]
       end
       assert_equal dir, sharder.persistence_path
@@ -97,15 +97,15 @@ id11,a11,b11
       size = 10
       sharder = Persist.tsv(dir,
                             persist_options: {
-                              :engine => 'fwt', 
-                              :path => dir, 
+                              :engine => 'fwt',
+                              :path => dir,
                               :serializer => :float,
-                              :range => false, :value_size => 64, 
-                              :shard_function => shard_function, 
+                              :range => false, :value_size => 64,
+                              :shard_function => shard_function,
                               :pos_function => pos_function
                             }
                            ) do |db|
-                             size.times do |v| 
+                             size.times do |v|
                                v = v + 1
                                chr = "chr" << (v % 5).to_s
                                key = chr + ":" << v.to_s
@@ -131,7 +131,7 @@ id11,a11,b11
 
       assert_equal 4.0, sharder["chr2:2"]
 
-      assert_equal size, sharder.size 
+      assert_equal size, sharder.size
     end
   end
 
@@ -147,19 +147,19 @@ id11,a11,b11
 
       size = 10
       chrs = (1..10).to_a
-      sharder = Persist.tsv(dir, 
-                            engine: 'pki', 
-                            :persist_options => { 
+      sharder = Persist.tsv(dir,
+                            engine: 'pki',
+                            :persist_options => {
                               :pattern =>  %w(f f),
-                              :range => false, 
-                              :value_size => 64, 
-                              :file => dir, 
+                              :range => false,
+                              :value_size => 64,
+                              :file => dir,
                               :shard_function => shard_function,
                               :pos_function => pos_function
                             }) do |db|
         chrs.each do |c|
-          size.times do |v| 
-            v = v 
+          size.times do |v|
+            v = v
             chr = "chr" << c.to_s
             key = chr + ":" << v.to_s
             db[key] = [v, v*2]
@@ -184,7 +184,7 @@ id11,a11,b11
       assert_equal count, size*chrs.length
 
       sharder = Persist.open_sharder(
-        dir, false, 'pki', 
+        dir, false, 'pki',
         {:pattern => %w(f f), :file => dir, :range => false, :value_size => 64, :pos_function => pos_function}, &shard_function
       )
 
@@ -209,7 +209,7 @@ id11,a11,b11
       chrs = (1..10).to_a
       sharder = Persist.tsv(dir, persist_options: {:pattern => %w(f), :range => false, :value_size => 64, :engine => 'pki', :file => dir, :shard_function => shard_function, :pos_function => pos_function}) do |db|
         chrs.each do |c|
-          size.times do |v| 
+          size.times do |v|
             v = v + 1
             chr = "chr" << c.to_s
             key = chr + ":" << (v*2).to_s
@@ -249,15 +249,15 @@ id11,a11,b11
 
       size = 10
       sharder = Persist.persist_tsv("ShardTSV_FWT", nil, {}, {
-                              :engine => 'fwt', 
-                              :path => dir, 
+                              :engine => 'fwt',
+                              :path => dir,
                               :serializer => :float,
-                              :range => false, :value_size => 64, 
-                              :shard_function => shard_function, 
+                              :range => false, :value_size => 64,
+                              :shard_function => shard_function,
                               :pos_function => pos_function
                             }
                            ) do |db|
-                             size.times do |v| 
+                             size.times do |v|
                                v = v + 1
                                chr = "chr" << (v % 5).to_s
                                key = chr + ":" << v.to_s
@@ -282,7 +282,7 @@ id11,a11,b11
 
       assert_equal 4.0, sharder["chr2:2"]
 
-      assert_equal size, sharder.size 
+      assert_equal size, sharder.size
     end
   end
 

data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb

@@ -112,7 +112,7 @@ row2    0.1  4.5 0
       assert_equal [0.2, 0.3, 0], tsv2["row1"]
       assert_equal TSVAdapter::FloatArraySerializer, tsv2.serializer
     end
- 
+
   end
 
   def test_float_double

data/test/scout/persist/tsv/test_serialize.rb

@@ -6,7 +6,7 @@ class TestClass < Test::Unit::TestCase
     m = TSVAdapter.serializer_module :marshal
     v = [1, :a]
     d = m.dump(v)
-    assert_equal v, m.load(d) 
+    assert_equal v, m.load(d)
   end
 end
 

data/test/scout/test_association.rb

@@ -40,7 +40,7 @@ class TestAssociation < Test::Unit::TestCase
     assert_equal "2021", database["Clei"]["Date"]
     assert_include database.key_field, "Alias"
   end
-  
+
   def test_brothers_id
     database = Association.database(datadir_test.person.brothers, :source => "Older=~Older (Alias)=>Name", :target => "Younger=~Younger (Alias)=>ID")
     assert_equal '001', database["Isabel"]["Younger"]

data/test/scout/test_tsv.rb

@@ -27,7 +27,7 @@ row4    a    a    id3
     tsv = TmpFile.with_file(content) do |filename|
       TSV.open(filename)
     end
-    
+
     TmpFile.with_file(content2) do |filename|
       TSV.open(filename, :data => tsv)
     end
@@ -228,7 +228,7 @@ row2    A AA AAA
 
   def test_flat_key
     content =<<-EOF
-#Id    ValueA 
+#Id    ValueA
 row1   a   aa   aaa
 row2   b  bbb bbbb bb aa
     EOF

data/test/scout/test_workflow.rb

@@ -42,7 +42,7 @@ class TestWorkflow < Test::Unit::TestCase
     end
 
     dep :prepare_batter
-    task :bake_muffin_tray => :string do 
+    task :bake_muffin_tray => :string do
       bake(step(:prepare_batter).load)
     end
 
@@ -53,7 +53,7 @@ class TestWorkflow < Test::Unit::TestCase
       batter = mix(whisked_eggs, Pantry.flour.produce)
 
       if add_bluberries
-        batter = mix(batter, Pantry.blueberries.produce) 
+        batter = mix(batter, Pantry.blueberries.produce)
       end
 
       batter

data/test/scout/tsv/test_annotation.rb

@@ -15,10 +15,10 @@ class TestAnnotationTSV < Test::Unit::TestCase
     AnnotationClass.setup(str1, :c11, :c12)
     AnnotationClass.setup(str2, :c21, :c22)
 
-    assert_equal str1, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str1.annotation_id + "#0"]["literal"] 
-    assert_equal :c11, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str1.annotation_id + "#0"]["code"] 
-    assert_equal str2, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str2.annotation_id + "#1"]["literal"] 
-    assert_equal :c21, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str2.annotation_id + "#1"]["code"] 
+    assert_equal str1, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str1.annotation_id + "#0"]["literal"]
+    assert_equal :c11, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str1.annotation_id + "#0"]["code"]
+    assert_equal str2, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str2.annotation_id + "#1"]["literal"]
+    assert_equal :c21, Annotation.tsv([str1, str2], :all).tap{|t| t.unnamed = false}[str2.annotation_id + "#1"]["code"]
     assert_equal "c11", JSON.parse(Annotation.tsv([str1, str2], :code, :JSON).tap{|t| t.unnamed = false}[str1.annotation_id + "#0"]["JSON"])["code"]
   end
 

data/test/scout/tsv/test_index.rb

@@ -15,7 +15,7 @@ class TestTSVIndex < Test::Unit::TestCase
     end
 
     tsv = tsv.slice ["Start", "End"]
- 
+
     tsv
   end
 

data/test/scout/tsv/test_open.rb

@@ -78,7 +78,7 @@ class TestOpenTraverse < Test::Unit::TestCase
     num_lines = 100
     lines = num_lines.times.collect{|i| "line-#{i}" }
 
-    assert_raise ScoutException do 
+    assert_raise ScoutException do
       i = 0
       Log.with_severity 7 do
         dumper = TSV::Dumper.new :key_field => "Key", :fields => ["Value"], :type => :single
@@ -125,7 +125,7 @@ row2 aa bb cc
 
     s = StringIO.new text
     collapsed = TSV.collapse_stream(s)
-    tsv = TSV.open collapsed 
+    tsv = TSV.open collapsed
     assert_equal ["A", "a"], tsv["row1"][0]
     assert_equal ["BB", "bb"], tsv["row2"][1]
   end

data/test/scout/tsv/test_parser.rb

@@ -15,7 +15,7 @@ class TestTSVParser < Test::Unit::TestCase
   def test_parse_line_key
     line = (0..10).to_a * "\t"
     key, values = TSV.parse_line(line, key: 2)
-    
+
     assert_equal "2", key
     assert_equal %w(0 1 3 4 5 6 7 8 9 10), values
   end
@@ -33,7 +33,7 @@ class TestTSVParser < Test::Unit::TestCase
     num = 10_000
     txt = num.times.inject(nil) do |acc,i|
       (acc.nil? ? "" : acc << "\n") << (0..10).collect{|v| v == 0 ? i : [v,v] * "|" } * "\t"
-    end 
+    end
 
     txt = StringIO.new(([txt] * (10))*"\n")
     Misc.benchmark 5 do

data/test/scout/tsv/test_stream.rb

@@ -14,7 +14,7 @@ row3 AAA BBB CCC
 
     text2=<<-EOF
 #: :sep=" "
-#Row Labela Labelb 
+#Row Labela Labelb
 row1 a b
 row2 aa bb
 row3 aaa bbb

data/test/scout/tsv/test_transformer.rb

@@ -26,7 +26,7 @@ row2    A2|A22    B2|B22
 
       assert_equal %w(A1 A11 B1 B11), tsv['row1']
     end
-    
+
   end
 
   def test_traverse

data/test/scout/tsv/util/test_filter.rb

@@ -182,7 +182,7 @@ row2    A    B
       tsv.pop_filter
       assert tsv.filename !~ /ValueA/
     end
- 
+
   end
 end
 

data/test/scout/tsv/util/test_melt.rb

@@ -11,7 +11,7 @@ class TestClass < Test::Unit::TestCase
 10,20,30
 20,40,60
     EOF
-    
+
     target =<<-EOF
 #: :sep=,#:type=:list
 #ID,Size,Time,System

data/test/scout/tsv/util/test_reorder.rb

@@ -83,7 +83,7 @@ row1    a    b    Id1
 row2    A    B    Id3
 row3    a    C    Id4
     EOF
- 
+
     TmpFile.with_file(content) do |filename|
       tsv = TSV.open(filename, :sep => /\s+/)
 

data/test/scout/work_queue/test_socket.rb

@@ -6,7 +6,7 @@ class TestSocket < Test::Unit::TestCase
   class DoneProcessing end
 
   def test_simple
-    socket = WorkQueue::Socket.new 
+    socket = WorkQueue::Socket.new
 
     socket.write 1
     socket.write 2
@@ -25,7 +25,7 @@ class TestSocket < Test::Unit::TestCase
   end
 
   def __test_speed
-    socket = WorkQueue::Socket.new 
+    socket = WorkQueue::Socket.new
 
     num = 50_000
 
@@ -33,7 +33,7 @@ class TestSocket < Test::Unit::TestCase
       num.times do |i|
         socket.write nil
       end
-      socket.write DoneProcessing.new 
+      socket.write DoneProcessing.new
     end
 
     bar = Log::ProgressBar.new num

data/test/scout/work_queue/test_worker.rb

@@ -46,7 +46,7 @@ class TestQueueWorker < Test::Unit::TestCase
           Open.consume_stream(sout, false, outfile)
           txt = Open.read(outfile)
           pid_list = txt.split("\n")
-          
+
           assert_equal (num_lines + 2) * num_workers, pid_list.length
 
           assert_nothing_raised do
@@ -122,7 +122,7 @@ class TestQueueWorker < Test::Unit::TestCase
       end
     end
 
-    read = Thread.new do 
+    read = Thread.new do
       begin
         while obj = output.read
           if DoneProcessing === obj

data/test/scout/workflow/deployment/orchestrator/test_batches.rb

@@ -35,7 +35,7 @@ module TestWFC
   dep :c1
   dep :c2
   task :c3 => :string do self.task_name.to_s end
-  
+
   dep_task :c4, TestWFC, :c3
 end
 
@@ -99,15 +99,25 @@ TestWFC:
     skip: true
     log: 4
     time: 10s
+  c4:
+    skip: true
   EOF
 
   def test_chain_batches
     job = TestWFD.job(:d1, nil)
     job.recursive_clean
 
-    job_chains_map = Workflow::Orchestrator.job_chains(RULES, job)
+    job_chain_list = []
+
+    job_chains = Workflow::Orchestrator.job_chains(RULES, job)
+    job_chains.each do |chain,list|
+      list.each do |info|
+        job_chain_list << [chain,info]
+      end
+    end
+
     workload = Workflow::Orchestrator.job_workload(job)
-    batches = Workflow::Orchestrator.chain_batches(RULES, job_chains_map, workload)
+    batches = Workflow::Orchestrator.chain_batches(RULES, job_chain_list, workload)
     assert_equal 3, batches.length
   end
 

data/test/scout/workflow/deployment/orchestrator/test_chains.rb

@@ -36,7 +36,7 @@ module TestWFC
   dep :c1
   dep :c2
   task :c3 => :string do self.task_name.to_s end
-  
+
   dep_task :c4, TestWFC, :c3
 end
 
@@ -110,11 +110,10 @@ TestWFC:
     job.recursive_clean
 
     job_chains = Workflow::Orchestrator.job_chains(RULES, job)
-    job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
 
-    assert_equal %w(chain_a_b chain_a), job_chains.fields
-    assert_equal 3, job_chains["chain_a"].last[:jobs].length
-    assert_equal job, job_chains["chain_a"].last[:top_level]
+    assert_equal %w(chain_a_b chain_a), job_chains.keys
+    assert_equal 3, job_chains["chain_a"].first[:jobs].length
+    assert_equal job, job_chains["chain_a"].first[:top_level]
   end
 
   def test_job_chain_b
@@ -122,9 +121,7 @@ TestWFC:
     job.recursive_clean
 
     job_chains = Workflow::Orchestrator.job_chains(RULES, job)
-    job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
 
-    assert_equal %w(chain_a chain_b chain_b2 chain_a_b).sort, job_chains.fields.sort
 
     assert_equal 2, job_chains["chain_a"].last[:jobs].length
     assert_equal job.step("a2"), job_chains["chain_a"].last[:top_level]
@@ -138,18 +135,23 @@ TestWFC:
     assert_equal 4, job_chains["chain_a_b"].last[:jobs].length
     assert_equal job, job_chains["chain_a_b"].last[:top_level]
   end
-  
+
   def test_job_chains_double
     job = TestWFD.job(:d1, nil)
     job.recursive_clean
 
+    rules = YAML.load <<-EOF
+chains:
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    EOF
+
     job_chains = Workflow::Orchestrator.job_chains(RULES, job)
-    job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
 
-    assert_equal 2, job_chains.select{|n,i| n == 'chain_a' }.length
-    assert_equal ["First c3", "Second c3"].sort, job_chains.
-      select{|n,i| n == 'chain_a' }.
-      collect{|n,i| i[:top_level].name }.sort
+    assert_equal 2, job_chains['chain_a'].length
+    assert_equal ["First c3", "Second c3"].sort, job_chains['chain_a'].
+      collect{|i| i[:top_level].name }.flatten.sort
 
     assert_equal 1, job_chains.select{|n,i| n == 'chain_d' }.length
   end

data/test/scout/workflow/deployment/orchestrator/test_workload.rb

@@ -34,7 +34,7 @@ module TestWFC
   dep :c1
   dep :c2
   task :c3 => :string do self.task_name.to_s end
-  
+
   dep_task :c4, TestWFC, :c3
 end
 

data/test/scout/workflow/deployment/test_local.rb

@@ -3,7 +3,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 require 'scout/workflow/deployment/trace'
 module TestWF
-  extend Workflow 
+  extend Workflow
   self.name = "TestWF"
 
   MULT = 0.1
@@ -83,7 +83,7 @@ TestWF:
       second_cpus[second] = cpus
     end
 
-    assert Misc.mean(second_cpus.values) > 15 
+    assert Misc.mean(second_cpus.values) > 15
     assert Misc.mean(second_cpus.values) < 30
   end
 

data/test/scout/workflow/deployment/test_scheduler.rb

@@ -6,7 +6,7 @@ require 'scout/workflow/deployment/scheduler/slurm'
 class TestScheduler < Test::Unit::TestCase
   setup do
     module TestWF
-      extend Workflow 
+      extend Workflow
       self.name = "TestWF"
 
       MULT ||= 0.1
@@ -69,7 +69,6 @@ TestWF:
 
     batches = Workflow::Orchestrator.job_batches(rules, jobs)
     dirs = Workflow::Scheduler.process_batches(batches, dry_run: true)
-    iii dirs
   end
 end
 

data/test/scout/workflow/step/test_children.rb

@@ -7,7 +7,7 @@ class TestStepChildren < Test::Unit::TestCase
   def test_child
     TmpFile.with_file do |tmpfile|
       step = Step.new tmpfile, ["12"] do |s|
-        pid = child do 
+        pid = child do
           Open.write(self.file(:somefile), 'TEST')
         end
         Process.waitpid pid

data/test/scout/workflow/step/test_dependencies.rb

@@ -60,7 +60,7 @@ class TestStepDependencies < Test::Unit::TestCase
       end
       step1.type = :array
 
-      step2 = Step.new tmpfile.step2 do 
+      step2 = Step.new tmpfile.step2 do
         step1 = dependencies.first
         raise ScoutException unless step1.streaming?
         stream = step1.stream
@@ -115,5 +115,40 @@ class TestStepDependencies < Test::Unit::TestCase
     assert_equal "2", step2.recursive_inputs["input1"]
     assert_equal "1", step3.recursive_inputs["input1"]
   end
+
+  def test_overriden_fixed
+    TmpFile.with_file("HELLO") do |file|
+      wf = Module.new do
+        extend Workflow
+
+        self.name = "TestWF"
+
+        task :message => :string do
+          "HI"
+        end
+
+        dep :message
+        task :say => :string do
+          "I say #{step(:message).load}"
+        end
+
+        task_alias :say_hello, self, :say, "TestWF#message" => file, :not_overriden => true
+      end
+
+      assert_equal "I say HI", wf.job(:say).run
+
+      job1 = wf.job(:say, "TestWF#message" => file)
+      assert_equal "I say HELLO", job1.run
+
+      job2 = wf.job(:say_hello)
+      assert_equal "I say HELLO", job2.run
+
+      assert job1.overriden?
+      refute job2.overriden?
+
+      assert job1.overrider_deps.any?
+      refute job2.overrider_deps.any?
+    end
+  end
 end
 

data/test/scout/workflow/step/test_info.rb

@@ -25,7 +25,7 @@ class TestStepInfo < Test::Unit::TestCase
       assert_equal 2, step1.exec
       assert_equal 2, step1.run
 
-      step2 = Step.new tmpfile.step2 do 
+      step2 = Step.new tmpfile.step2 do
         step1 = dependencies.first
         step1.inputs.first + " has " + step1.load.to_s + " characters"
       end
@@ -49,6 +49,8 @@ class TestStepInfo < Test::Unit::TestCase
 
       step1.run
 
+      step1 = Step.new tmpfile.step1
+
       refute Path === step1.info[:inputs][0]
 
     end
@@ -70,38 +72,4 @@ class TestStepInfo < Test::Unit::TestCase
     end
   end
 
-  def test_overriden_fixed
-    TmpFile.with_file("HELLO") do |file|
-      wf = Module.new do
-        extend Workflow
-
-        self.name = "TestWF"
-
-        task :message => :string do
-          "HI"
-        end
-
-        dep :message
-        task :say => :string do
-          "I say #{step(:message).load}"
-        end
-
-        task_alias :say_hello, self, :say, "TestWF#message" => file, :not_overriden => true
-      end
-
-      assert_equal "I say HI", wf.job(:say).run
-
-      job1 = wf.job(:say, "TestWF#message" => file)
-      assert_equal "I say HELLO", job1.run
-
-      job2 = wf.job(:say_hello)
-      assert_equal "I say HELLO", job2.run
-
-      assert job1.overriden?
-      refute job2.overriden?
-
-      assert job1.overriden_deps.any?
-      refute job2.overriden_deps.any?
-    end
-  end
 end

data/test/scout/workflow/step/test_load.rb

@@ -10,7 +10,7 @@ class TestStepLoad < Test::Unit::TestCase
       s.length
     end
 
-    step2 = Step.new tmpfile.step2 do 
+    step2 = Step.new tmpfile.step2 do
       step1 = dependencies.first
       step1.inputs.first + " has " + step1.load.to_s + " characters"
     end