scout-gear 7.1.0 → 7.3.0

data/test/scout/tsv/test_traverse.rb

@@ -0,0 +1,116 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestTSVTraverse < Test::Unit::TestCase
+  def test_tsv_traverse_double
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    AA    BB    Id33
+    EOF
+
+    sss 0
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    res = {}
+    tsv.traverse :key, %w(OtherID ValueB) do |k,v|
+      res[k] = v
+    end
+    assert_equal [%w(Id3 Id33), %w(B BB)], res["row2"]
+
+    res = {}
+    tsv.traverse :key, %w(OtherID ValueB), type: :list do |k,v|
+      res[k] = v
+    end
+    assert_equal ["Id3", "B"], res["row2"]
+
+    res = {}
+    tsv.traverse "OtherID", %w(Id ValueB), one2one: true do |k,v|
+      res[k] = v
+    end
+    assert_equal [[nil], %w(BB)], res["Id33"]
+
+    res = {}
+    tsv.traverse "OtherID", %w(Id ValueB), one2one: true, type: :list do |k,v|
+      res[k] = v
+    end
+    assert_equal ["row2", "B"], res["Id3"]
+    assert_equal [nil, "BB"], res["Id33"]
+
+    tsv.traverse "OtherID", %w(Id ValueB), one2one: false, type: :list do |k,v|
+      res[k] = v
+    end
+    assert_equal ["row2", "B"], res["Id3"]
+    assert_equal [nil, "BB"], res["Id33"]
+
+    res = {}
+    key_name, field_names = tsv.traverse "OtherID" do |k,v|
+      res[k] = v
+    end
+    assert_equal "OtherID", key_name
+    assert_equal %w(Id ValueA ValueB), field_names
+  end
+
+  def test_tsv_traverse_list
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:list
+#Id    ValueA    ValueB    OtherID
+row1    a    b    Id1
+row2    A    B    Id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    res = {}
+    tsv.traverse :key, %w(OtherID ValueB) do |k,v|
+      res[k] = v
+    end
+    assert_equal ["Id3", "B"], res["row2"]
+
+    res = {}
+    tsv.traverse :key, %w(OtherID ValueB), type: :double do |k,v|
+      res[k] = v
+    end
+    assert_equal [%w(Id3), %w(B)], res["row2"]
+
+    res = {}
+    tsv.traverse :key, %w(OtherID ValueB), type: :flat do |k,v|
+      res[k] = v
+    end
+    assert_equal %w(Id3 B), res["row2"]
+  end
+
+  def test_tsv_traverse_single
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:single
+#Id    ValueA
+row1    a
+row2    A
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    res = {}
+    tsv.traverse "ValueA", %w(Id) do |k,v|
+      res[k] = v
+    end
+    assert_equal "row1", res["a"]
+
+    res = {}
+    tsv.traverse "ValueA", %w(Id), type: :double do |k,v|
+      res[k] = v
+    end
+    assert_equal [["row1"]], res["a"]
+  end
+end
+

data/test/scout/tsv/test_util.rb

@@ -0,0 +1,23 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVUtil < Test::Unit::TestCase
+  def test_open_persist
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double#:merge=:concat
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    a    a    id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :sep => " " )
+    end
+    assert_equal %w(row1 row2), tsv.collect{|k,v| k }
+    assert NamedArray === tsv.collect{|k,v| v }.first
+  end
+
+end
+

data/test/scout/tsv/util/test_filter.rb

@@ -0,0 +1,188 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVFilters < Test::Unit::TestCase
+  def test_collect
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      assert_equal 2, tsv.collect.size
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"]
+      assert_equal 1, tsv.collect.size
+    end
+  end
+
+  def test_through
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"]
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 1, elem.size
+    end
+  end
+
+  def test_2_filters
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+row3    A    C
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"]
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 2, elem.size
+      assert_equal ["row2", "row3"], elem.sort
+
+      tsv.add_filter "field:ValueB", ["C"]
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 1, elem.size
+    end
+  end
+
+  def test_filter_persistence
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+row3    A    C
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"], tmpdir.fieldValueA.find(:user)
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 2, elem.size
+
+      tsv.add_filter "field:ValueB", ["C"]
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 1, elem.size
+
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"], tmpdir.fieldValueA.find(:user)
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 2, elem.size
+    end
+  end
+
+  def __test_filter_persistence_update
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"], Scout.tmp.test.Filter.fieldValueA.find(:user)
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 1, elem.size
+
+      tsv["row3"] = [["A"], ["C"]]
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 2, elem.size
+
+      tsv.add_filter "field:ValueB", ["C"]
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 1, elem.size
+
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"], Scout.tmp.test.Filter.fieldValueA.find(:user)
+
+      elem = []
+      tsv.through do |k,v| elem << k end
+      assert_equal 1, elem.size
+    end
+  end
+
+  def test_delete
+    content =<<-EOF
+#ID ValueA ValueB Comment
+row1 a b c
+row2 A B C
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), type: :double, :sep => /\s/)
+      assert_equal 2, tsv.keys.length
+      tsv.delete "row2"
+      assert_equal 1, tsv.keys.length
+
+      tsv = TSV.open(File.open(filename), type: :double, :sep => /\s/)
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"]
+
+      assert_equal 1, tsv.keys.length
+      assert_equal ["row2"], tsv.keys
+
+      tsv.delete "row2"
+      assert_equal 0, tsv.keys.length
+
+      tsv.pop_filter
+      assert_equal ["row1"], tsv.keys
+    end
+  end
+
+  def test_filename
+    content1 =<<-EOF
+#: :sep=/\\s+/#:case_insensitive=false
+#Id    ValueA    ValueB
+row1    a|aa|aaa    b
+row2    A    B
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      tsv = TSV.open filename
+      tsv.filter
+      tsv.add_filter "field:ValueA", ["A"]
+      assert tsv.filename =~ /ValueA/
+      tsv.pop_filter
+      assert tsv.filename !~ /ValueA/
+    end
+ 
+  end
+end
+

data/test/scout/tsv/util/test_process.rb

@@ -0,0 +1,47 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVProcess < Test::Unit::TestCase
+  def test_process
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    AA    BB    Id33
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename)
+    end
+
+    tsv.process "ValueA" do |v|
+      v.collect{|e| e.upcase }
+    end
+
+    assert_equal %w(A AA AAA), tsv["row1"][0]
+  end
+
+  def test_add_field
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    AA    BB    Id33
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename)
+    end
+
+    tsv.add_field "ValueC" do |k,v|
+      v[0].collect{|e| e.gsub("a", "c").gsub("A", "C") }
+    end
+
+    assert_equal %w(c cc ccc), tsv["row1"]["ValueC"]
+    assert_equal %w(C CC), tsv["row2"]["ValueC"]
+  end
+end
+

data/test/scout/tsv/util/test_select.rb

@@ -0,0 +1,44 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVSelect < Test::Unit::TestCase
+  def test_select
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    AA    BB    Id33
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    s = tsv.select do |k,v|
+      k.include? "2"
+    end
+
+    assert_equal ['row2'], s.keys
+  end
+
+  def test_reorder
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a1|a2    b1|b2    Id1|Id2
+row2    A1|A3    B1|B3    Id1|Id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename)
+    end
+
+    r = tsv.reorder "OtherID", %w(ValueB Id)
+
+    assert_equal %w(row1 row2), r["Id1"]["Id"]
+    assert_equal %w(row2), r["Id3"]["Id"]
+  end
+end
+

data/test/scout/work_queue/test_worker.rb

@@ -16,34 +16,91 @@ class TestQueueWorker < Test::Unit::TestCase
     end
   end
 
-  def test_semaphore
-    ScoutSemaphore.with_semaphore 1 do |sem|
+  def test_semaphore_pipe
+
+    2.times do
+      num_lines = 10
+      num_workers = 100
 
       TmpFile.with_file do |outfile|
-        2.times do
+        Open.rm(outfile)
+        ScoutSemaphore.with_semaphore 1 do |sem|
           sout = Open.open_pipe do |sin|
-            workers = 100.times.collect{ WorkQueue::Worker.new }
+            workers = num_workers.times.collect{ WorkQueue::Worker.new }
             workers.each do |w|
               w.run do
                 ScoutSemaphore.synchronize(sem) do
-                  10.times do
-                    sin.puts Process.pid
+                  sin.puts "Start - #{Process.pid}"
+                  num_lines.times do |i|
+                    sin.puts "line-#{i}-#{Process.pid}"
                   end
+                  sin.puts "End - #{Process.pid}"
                 end
               end
             end
+            sin.close
 
             WorkQueue::Worker.join(workers)
           end
+
           Open.consume_stream(sout, false, outfile)
+          txt = Open.read(outfile)
+          pid_list = txt.split("\n")
+          
+          assert_equal (num_lines + 2) * num_workers, pid_list.length
+
+          assert_nothing_raised do
+            seen = []
+            current = nil
+            pid_list.each do |pid|
+              if pid != current
+                raise "Out of order #{Log.fingerprint seen} #{ pid }" if seen.include? pid
+              end
+              current = pid
+              seen << pid
+            end
+          end
+        end
+      end
+    end
+  end
+  def test_semaphore
+
+    2.times do
+      num_lines = 10
+      num_workers = 500
+
+      TmpFile.with_file do |outfile|
+        Open.rm(outfile)
+        ScoutSemaphore.with_semaphore 1 do |sem|
+          workers = num_workers.times.collect{ WorkQueue::Worker.new }
+          Open.touch(outfile)
+          workers.each do |w|
+            w.run do
+              ScoutSemaphore.synchronize(sem) do
+                sin = Open.open(outfile, :mode => 'a')
+                sin.puts "Start - #{Process.pid}"
+                  num_lines.times do |i|
+                    sin.puts "line-#{i}-#{Process.pid}"
+                  end
+                sin.puts "End - #{Process.pid}"
+                sin.close
+              end
+            end
+          end
+
+          WorkQueue::Worker.join(workers)
+
+
           pid_list = Open.read(outfile).split("\n")
+          assert_equal (num_lines + 2) * num_workers, pid_list.length
 
           assert_nothing_raised do
             seen = []
             current = nil
             pid_list.each do |pid|
               if pid != current
-                raise "Out of order" if seen.include? pid
+                raise "Out of order #{Log.fingerprint seen} #{ pid }" if seen.include? pid
               end
               current = pid
               seen << pid
@@ -113,7 +170,7 @@ class TestQueueWorker < Test::Unit::TestCase
         while obj = output.read
           if DoneProcessing === obj
             pid = obj.pid
-            workers.delete_if{|w| w.pid = pid }
+            @worker_mutex.synchronize{ @workers.delete_if{|w| w.pid = pid } }
             break if workers.empty?
           end
           raise obj if Exception === obj
@@ -134,7 +191,7 @@ class TestQueueWorker < Test::Unit::TestCase
 
     write.join
 
-    assert_raise ScoutException do
+    assert_raise WorkerException do
       read.join
     end
 

data/test/scout/workflow/step/test_dependencies.rb

@@ -0,0 +1,25 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/named_array'
+class TestStepDependencies < Test::Unit::TestCase
+  def test_recursive_inputs
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, NamedArray.setup(["12"], %w(input1)) do |s|
+      s.length
+    end
+
+    step2 = Step.new tmpfile.step2, NamedArray.setup([2], %w(input2)) do |times|
+      step1 = dependencies.first
+      (step1.inputs.first + " has " + step1.load.to_s + " characters") * times
+    end
+
+    step2.dependencies = [step1]
+
+    assert_equal 2, step2.inputs["input2"]
+    assert_equal "12", step2.recursive_inputs["input1"]
+    assert_equal 2, step2.inputs[:input2]
+    assert_equal "12", step2.recursive_inputs[:input1]
+  end
+end
+

data/test/scout/workflow/step/test_info.rb

@@ -4,27 +4,25 @@ require 'scout/workflow'
 
 class TestStepInfo < Test::Unit::TestCase
   def test_dependency
-    sss 0 do
-      TmpFile.with_file do |tmpdir|
-        Path.setup(tmpdir)
-        tmpfile = tmpdir.test_step
-        step1 = Step.new tmpfile.step1, ["12"] do |s|
-          s.length
-        end
+    TmpFile.with_file do |tmpdir|
+      Path.setup(tmpdir)
+      tmpfile = tmpdir.test_step
+      step1 = Step.new tmpfile.step1, ["12"] do |s|
+        s.length
+      end
 
-        assert_equal 2, step1.exec
-        assert_equal 2, step1.run
+      assert_equal 2, step1.exec
+      assert_equal 2, step1.run
 
-        step2 = Step.new tmpfile.step2 do 
-          step1 = dependencies.first
-          step1.inputs.first + " has " + step1.load.to_s + " characters"
-        end
+      step2 = Step.new tmpfile.step2 do 
+        step1 = dependencies.first
+        step1.inputs.first + " has " + step1.load.to_s + " characters"
+      end
 
-        step2.dependencies = [step1]
+      step2.dependencies = [step1]
 
-        assert_equal "12 has 2 characters", step2.run
-        assert_equal "12 has 2 characters", step2.run
-      end
+      assert_equal "12 has 2 characters", step2.run
+      assert_equal "12 has 2 characters", step2.run
     end
   end
 end

data/test/scout/workflow/step/test_load.rb

@@ -5,29 +5,27 @@ require 'scout/workflow/step'
 
 class TestStepLoad < Test::Unit::TestCase
   def test_dependency
-    sss 0 do
-      tmpfile = tmpdir.test_step
-      step1 = Step.new tmpfile.step1, ["12"] do |s|
-        s.length
-      end
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, ["12"] do |s|
+      s.length
+    end
 
-      step2 = Step.new tmpfile.step2 do 
-        step1 = dependencies.first
-        step1.inputs.first + " has " + step1.load.to_s + " characters"
-      end
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      step1.inputs.first + " has " + step1.load.to_s + " characters"
+    end
 
-      step2.dependencies = [step1]
+    step2.dependencies = [step1]
 
-      step2.recursive_clean
-      step2.run
+    step2.recursive_clean
+    step2.run
 
-      new_step2 = Step.load(step2.path)
+    new_step2 = Step.load(step2.path)
 
-      assert_equal "12 has 2 characters", new_step2.load
-      assert_equal "12 has 2 characters", new_step2.run
-      assert_equal 2, new_step2.dependencies.first.run
-      assert_equal "12", new_step2.dependencies.first.inputs.first
-    end
+    assert_equal "12 has 2 characters", new_step2.load
+    assert_equal "12 has 2 characters", new_step2.run
+    assert_equal 2, new_step2.dependencies.first.run
+    assert_equal "12", new_step2.dependencies.first.inputs.first
   end
 
   def test_relocate
@@ -43,7 +41,7 @@ class TestStepLoad < Test::Unit::TestCase
       end
     end
 
-    step2 = wf.job(:step2, :input1 => "TEST")
+    step2 = wf.job(:step2, nil, :input1 => "TEST_STRING")
     step1 = step2.step(:step1)
 
     step2.run
@@ -55,8 +53,8 @@ class TestStepLoad < Test::Unit::TestCase
         Open.mv step1.info_file, dir.var.jobs.RelocateWorkflow.step1[File.basename(step1.info_file)]
 
         new_step2 = Step.load(step2.path)
-        assert_equal "TEST".reverse, new_step2.load
-        assert_equal "TEST", new_step2.dependencies.first.load
+        assert_equal "TEST_STRING".reverse, new_step2.load
+        assert_equal "TEST_STRING", new_step2.dependencies.first.load
       end
     end
 

data/test/scout/workflow/step/test_provenance.rb

@@ -0,0 +1,25 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/workflow'
+class TestStepProvenance < Test::Unit::TestCase
+  def test_true
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, ["12"] do |s|
+      s.length
+    end
+
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      step1.inputs.first + " has " + step1.load.to_s + " characters"
+    end
+
+    step2.dependencies = [step1]
+
+    step2.run
+
+    assert_include Step.prov_report(step2), 'step1'
+    assert_include Step.prov_report(step2), 'step2'
+  end
+end
+