scout-gear 7.2.0 → 8.0.0

data/test/scout/tsv/util/test_reorder.rb

@@ -0,0 +1,94 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+
+class TestReorder < Test::Unit::TestCase
+  def test_reorder
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a1|a2    b1|b2    Id1|Id2
+row2    A1|A3    B1|B3    Id1|Id3
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename)
+    end
+
+    r = tsv.reorder "OtherID", %w(ValueB Id)
+
+    assert_equal %w(row1 row2), r["Id1"]["Id"]
+    assert_equal %w(row2), r["Id3"]["Id"]
+  end
+
+  def test_reorder_list
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:list
+#Id    ValueA    ValueB    OtherID
+row1    a1    b1    Id1
+row2    A1    B1    Id1
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename)
+    end
+
+    r = tsv.reorder "ValueB"
+
+    assert_equal "row1", r["b1"]["Id"]
+    assert_equal "row2", r["B1"]["Id"]
+  end
+
+  def test_reorder_single
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:single
+#Id    ValueA
+row1    a1
+row2    A1
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename)
+    end
+
+    r = tsv.reorder "ValueA"
+
+    assert_equal "row1", r["a1"]
+    assert_equal "row2", r["A1"]
+  end
+
+  def test_transpose
+     content =<<-EOF
+#: :type=:list
+#Row   vA   vB   vID
+row1    a    b    Id1
+row2    A    B    Id3
+row3    a    C    Id4
+    EOF
+ 
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/)
+
+      assert_equal %w(vA vB vID),  tsv.transpose("Values").keys
+      assert_equal %w(Id1 Id3 Id4),  tsv.transpose("Values")["vID"]
+    end
+  end
+
+  def test_column
+    content =<<-EOF
+#Id    ValueA    ValueB ValueC
+rowA    A|AA    B|BB  C|CC
+rowa    a|aa    b|BB  C|CC
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :double)
+      tsv = tsv.column("ValueA", cast: :downcase)
+      assert_equal %w(a aa), tsv["rowA"]
+      assert_equal %w(a aa), tsv["rowa"]
+    end
+  end
+
+end
+

data/test/scout/tsv/util/test_select.rb

@@ -0,0 +1,58 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVSelect < Test::Unit::TestCase
+  def test_select
+    content =<<-'EOF'
+#: :sep=/\s+/#:type=:double
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row2    AA    BB    Id33
+    EOF
+
+    tsv = TmpFile.with_file(content) do |filename|
+      TSV.open(filename, :persist => true)
+    end
+
+    s = tsv.select do |k,v|
+      k.include? "2"
+    end
+
+    assert_equal ['row2'], s.keys
+  end
+
+  def test_select_values
+    content =<<-'EOF'
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    tsv = TSV.open(content)
+
+    assert TSV.select("row1", tsv["row1"], "A1")
+    refute TSV.select("row2", tsv["row2"], "A1")
+    assert TSV.select("row2", tsv["row2"], "A2")
+
+    assert TSV.select("row1", tsv["row1"], "B1")
+    refute TSV.select("row2", tsv["row2"], "B1")
+
+    refute TSV.select("row1", tsv["row1"], "B1", invert: true)
+    assert TSV.select("row2", tsv["row2"], "B1", invert: true)
+
+    assert TSV.select("row1", tsv["row1"], {"ValueB" => "B1"})
+    refute TSV.select("row2", tsv["row2"], {"ValueB" => "B1"})
+
+    tsv.with_unnamed do
+      assert TSV.select("row1", tsv["row1"], {"ValueB" => "B1"}, fields: tsv.fields)
+      refute TSV.select("row2", tsv["row2"], {"ValueB" => "B1"}, fields: tsv.fields)
+    end
+
+    assert TSV.select("row1", tsv["row1"], "B1", field: "ValueB")
+    refute TSV.select("row2", tsv["row2"], "B1", field: "ValueB")
+  end
+end
+

data/test/scout/tsv/util/test_unzip.rb

@@ -0,0 +1,112 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVUnzip < Test::Unit::TestCase
+  def test_unzip
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    tsv = TSV.open(content1)
+    unzip = tsv.unzip("ValueA", delete: true)
+    assert_equal "B1", unzip["row1:A1"]["ValueB"]
+    assert_equal "B22", unzip["row2:A22"]["ValueB"]
+  end
+
+  def test_unzip_list
+    content1 =<<-EOF
+#: :sep=" "#:type=:list
+#ID    ValueA    ValueB
+row1    A1    B1
+row2    A2    B2
+    EOF
+
+    tsv = TSV.open(content1)
+    unzip = tsv.unzip("ValueA", delete: true)
+    assert_equal "B1", unzip["row1:A1"]["ValueB"]
+    assert_equal "B2", unzip["row2:A2"]["ValueB"]
+  end
+
+  def test_unzip_merge
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11|A11    B1|B11|B11.1
+row2    A2|A22|A22    B2|B22|B22.2
+    EOF
+
+    tsv = TSV.open(content1)
+    unzip = tsv.unzip("ValueA", delete: true, merge: true)
+    assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
+    assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
+    assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
+  end
+
+  def test_unzip_merge_stream
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11|A11    B1|B11|B11.1
+row2    A2|A22|A22    B2|B22|B22.2
+    EOF
+
+    tsv = TSV.open(content1)
+    unzip = tsv.unzip("ValueA", delete: true, merge: true, target: :stream).tsv
+    assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
+    assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
+    assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
+  end
+
+  def test_unzip_merge_stream_parser
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11|A11    B1|B11|B11.1
+row2    A2|A22|A22    B2|B22|B22.2
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      unzip = TSV.unzip(filename, "ValueA", delete: true, merge: true, target: :stream).tsv
+      assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
+      assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
+      assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
+    end
+  end
+
+  def test_unzip_merge_stream_parser_one2many
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11|A11    B1|B11|B11.1
+row2    A2|A22|A22    B2|B22|B22.2
+    EOF
+
+    TmpFile.with_file(content1) do |filename|
+      unzip = TSV.unzip(filename, "ValueA", delete: true, merge: true, target: :stream, one2one: false).tsv
+      assert_equal ["B1", "B11", "B11.1"], unzip["row1:A1"]["ValueB"].uniq
+      assert_equal ["B1", "B11", "B11.1"], unzip["row1:A11"]["ValueB"].uniq
+    end
+  end
+
+
+  def test_unzip_target
+    content1 =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11|A11    B1|B11|B11.1
+row2    A2|A22|A22    B2|B22|B22.2
+    EOF
+
+    tsv = TSV.open(content1)
+    unzip = TSV.setup({}, :key_field => "Key", :fields => %w(ValueA ValueB))
+    tsv.unzip("ValueA", delete: true, merge: true, target: unzip)
+    assert_equal ["B1"], unzip["row1:A1"]["ValueB"]
+    assert_equal ["B11", "B11.1"], unzip["row1:A11"]["ValueB"]
+    assert_equal ["B22", "B22.2"], unzip["row2:A22"]["ValueB"]
+  end
+end
+

data/test/scout/work_queue/test_socket.rb

@@ -22,7 +22,6 @@ class TestSocket < Test::Unit::TestCase
   end
 
   def __test_speed
-    sss 0
     socket = WorkQueue::Socket.new 
 
     num = 10_000

data/test/scout/work_queue/test_worker.rb

@@ -16,27 +16,84 @@ class TestQueueWorker < Test::Unit::TestCase
     end
   end
 
-  def test_semaphore
-    ScoutSemaphore.with_semaphore 1 do |sem|
+  def test_semaphore_pipe
+
+    2.times do
+      num_lines = 10
+      num_workers = 100
 
       TmpFile.with_file do |outfile|
-        2.times do
+        Open.rm(outfile)
+        ScoutSemaphore.with_semaphore 1 do |sem|
           sout = Open.open_pipe do |sin|
-            workers = 100.times.collect{ WorkQueue::Worker.new }
+            workers = num_workers.times.collect{ WorkQueue::Worker.new }
             workers.each do |w|
               w.run do
                 ScoutSemaphore.synchronize(sem) do
-                  10.times do
-                    sin.puts Process.pid
+                  sin.puts "Start - #{Process.pid}"
+                  num_lines.times do |i|
+                    sin.puts "line-#{i}-#{Process.pid}"
                   end
+                  sin.puts "End - #{Process.pid}"
                 end
               end
             end
+            sin.close
 
             WorkQueue::Worker.join(workers)
           end
+
           Open.consume_stream(sout, false, outfile)
+          txt = Open.read(outfile)
+          pid_list = txt.split("\n")
+          
+          assert_equal (num_lines + 2) * num_workers, pid_list.length
+
+          assert_nothing_raised do
+            seen = []
+            current = nil
+            pid_list.each do |pid|
+              if pid != current
+                raise "Out of order #{Log.fingerprint seen} #{ pid }" if seen.include? pid
+              end
+              current = pid
+              seen << pid
+            end
+          end
+        end
+      end
+    end
+  end
+  def test_semaphore
+
+    2.times do
+      num_lines = 10
+      num_workers = 500
+
+      TmpFile.with_file do |outfile|
+        Open.rm(outfile)
+        ScoutSemaphore.with_semaphore 1 do |sem|
+          workers = num_workers.times.collect{ WorkQueue::Worker.new }
+          Open.touch(outfile)
+          workers.each do |w|
+            w.run do
+              ScoutSemaphore.synchronize(sem) do
+                sin = Open.open(outfile, :mode => 'a')
+                sin.puts "Start - #{Process.pid}"
+                  num_lines.times do |i|
+                    sin.puts "line-#{i}-#{Process.pid}"
+                  end
+                sin.puts "End - #{Process.pid}"
+                sin.close
+              end
+            end
+          end
+
+          WorkQueue::Worker.join(workers)
+
+
           pid_list = Open.read(outfile).split("\n")
+          assert_equal (num_lines + 2) * num_workers, pid_list.length
 
           assert_nothing_raised do
             seen = []

data/test/scout/workflow/step/test_load.rb

@@ -41,7 +41,7 @@ class TestStepLoad < Test::Unit::TestCase
       end
     end
 
-    step2 = wf.job(:step2, :input1 => "TEST")
+    step2 = wf.job(:step2, nil, :input1 => "TEST_STRING")
     step1 = step2.step(:step1)
 
     step2.run
@@ -53,8 +53,8 @@ class TestStepLoad < Test::Unit::TestCase
         Open.mv step1.info_file, dir.var.jobs.RelocateWorkflow.step1[File.basename(step1.info_file)]
 
         new_step2 = Step.load(step2.path)
-        assert_equal "TEST".reverse, new_step2.load
-        assert_equal "TEST", new_step2.dependencies.first.load
+        assert_equal "TEST_STRING".reverse, new_step2.load
+        assert_equal "TEST_STRING", new_step2.dependencies.first.load
       end
     end
 

data/test/scout/workflow/step/test_status.rb

@@ -0,0 +1,31 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestStepStatus < Test::Unit::TestCase
+  def test_dependency
+    tmpfile = tmpdir.test_step
+    step1 = Step.new tmpfile.step1, ["12"] do |s|
+      s.length
+    end
+
+    step2 = Step.new tmpfile.step2 do 
+      step1 = dependencies.first
+      step1.inputs.first + " has " + step1.load.to_s + " characters"
+    end
+
+    step2.dependencies = [step1]
+
+    sss 0
+    Misc.with_env "SCOUT_UPDATE", "true" do
+      step2.run
+      assert step2.updated?
+
+      sleep 0.1
+      step1.clean
+      step1.run
+      refute step2.updated?
+    end
+  end
+
+end
+

data/test/scout/workflow/task/test_inputs.rb

@@ -71,8 +71,8 @@ class TestTaskInput < Test::Unit::TestCase
     task = self.example_task
 
     TmpFile.with_file("2\n3") do |integer_array_file|
-      assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
-        task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
+      assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
+        task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
     end
   end
 
@@ -83,10 +83,10 @@ class TestTaskInput < Test::Unit::TestCase
 
     TmpFile.with_file("1\n2") do |integer_array_file|
       hash1 = Open.open(integer_array_file) do |f|
-        task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
+        task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
       end
       hash2 = Open.open(integer_array_file) do |f|
-        task.digest_inputs(:string => "String", :integer => 2, :integer_array => f)
+        task.process_inputs(:string => "String", :integer => 2, :integer_array => f).last
       end
       assert_equal hash1, hash2
     end
@@ -98,8 +98,8 @@ class TestTaskInput < Test::Unit::TestCase
     task = self.example_task
 
     TmpFile.with_file("2\n3") do |integer_array_file|
-      assert_equal task.digest_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)),
-        task.digest_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)
+      assert_equal task.process_inputs(:string => "String", :integer => 2, :integer_array => %w(2 3)).last,
+        task.process_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file).last
     end
   end
 
@@ -108,12 +108,12 @@ class TestTaskInput < Test::Unit::TestCase
 
     TmpFile.with_file("2\n3") do |integer_array_file|
       inputs = {:string => "String", :integer => 2, :integer_array => integer_array_file, :float_array => %w(1.1 2.2)}
-      original_digest =  task.digest_inputs(inputs)
+      original_digest =  task.process_inputs(inputs).last
 
       TmpFile.with_file do |save_directory|
         task.save_inputs(save_directory, inputs)
         new_inputs = task.load_inputs(save_directory)
-        new_digest =  task.digest_inputs(new_inputs)
+        new_digest =  task.process_inputs(new_inputs).last
         assert_equal original_digest, new_digest
       end
     end
@@ -124,13 +124,13 @@ class TestTaskInput < Test::Unit::TestCase
 
     TmpFile.with_file("TEST") do |somefile|
       inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
-      original_digest =  task.digest_inputs(inputs)
+      original_digest =  task.process_inputs(inputs).last
 
       TmpFile.with_file do |save_directory|
         task.save_inputs(save_directory, inputs)
         Open.rm somefile
         new_inputs = task.load_inputs(save_directory)
-        new_digest =  task.digest_inputs(new_inputs)
+        new_digest =  task.process_inputs(new_inputs).last
         assert_equal original_digest, new_digest
       end
     end
@@ -141,7 +141,7 @@ class TestTaskInput < Test::Unit::TestCase
 
     TmpFile.with_file("TEST") do |somefile|
       inputs = {:string => "String", :integer => 2, :file => somefile, :float_array => %w(1.1 2.2)}
-      original_digest =  task.digest_inputs(inputs)
+      original_digest =  task.process_inputs(inputs).last
 
       TmpFile.with_file do |save_directory|
         task.save_inputs(save_directory, inputs)
@@ -149,7 +149,7 @@ class TestTaskInput < Test::Unit::TestCase
           Open.cp save_directory, copy_directory
           Open.rm_rf save_directory
           new_inputs = task.load_inputs(copy_directory)
-          new_digest =  task.digest_inputs(new_inputs)
+          new_digest =  task.process_inputs(new_inputs).last
           assert_equal original_digest, new_digest
         end
       end
@@ -167,14 +167,14 @@ class TestTaskInput < Test::Unit::TestCase
       Open.write(file1, "TEST1")
       Open.write(file2, "TEST2")
       inputs = {:string => "String", :integer => 2, :file_array => [file1, file2], :float_array => %w(1.1 2.2)}
-      original_digest =  task.digest_inputs(inputs)
+      original_digest =  task.process_inputs(inputs).last
 
       TmpFile.with_file do |save_directory|
         task.save_inputs(save_directory, inputs)
         Open.rm(file1)
         Open.rm(file2)
         new_inputs = task.load_inputs(save_directory)
-        new_digest =  task.digest_inputs(new_inputs)
+        new_digest =  task.process_inputs(new_inputs).last
         assert_equal original_digest, new_digest
       end
     end

data/test/scout/workflow/test_step.rb

@@ -27,7 +27,6 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step2.dependencies = [step1]
 
-
     assert_equal "12 has 2 characters", step2.run
     assert_equal "12 has 2 characters", step2.run
   end
@@ -50,7 +49,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step2 = Step.new tmpfile.step2 do 
       step1 = dependencies.first
-      stream = step1.get_stream
+      stream = step1.stream
 
       Open.open_pipe do |sin|
         while line = stream.gets
@@ -76,7 +75,7 @@ class TestWorkflowStep < Test::Unit::TestCase
     end
     assert_equal times/2, lines.length
 
-    stream = step2.run
+    stream = step2.run(true)
     assert step1.streaming?
     assert step2.streaming?
 
@@ -87,7 +86,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     assert step1.path.read.end_with? "line-#{times-1}\n"
     assert_equal times/2, lines.length
-    assert_equal times/2, step2.path.read.split("\n").length
+    assert_equal times/2, step2.join.path.read.split("\n").length
   end
 
   def test_streaming_duplicate
@@ -108,7 +107,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step2 = Step.new tmpfile.step2 do 
       step1 = dependencies.first
-      stream = step1.get_stream
+      stream = step1.stream
 
       Open.open_pipe do |sin|
         while line = stream.gets
@@ -123,7 +122,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step3 = Step.new tmpfile.step3 do 
       step1, step2 = dependencies
-      stream = step2.get_stream
+      stream = step2.stream
 
       Open.open_pipe do |sin|
         while line = stream.gets
@@ -138,7 +137,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step3.recursive_clean
 
-    stream = step3.run
+    stream = step3.run(true)
     out = []
     while l = stream.gets
       out << l
@@ -164,7 +163,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step2 = Step.new tmpfile.step2 do 
       step1 = dependencies.first
-      stream = step1.get_stream
+      stream = step1.stream
 
       Open.open_pipe do |sin|
         while line = stream.gets
@@ -179,7 +178,7 @@ class TestWorkflowStep < Test::Unit::TestCase
 
     step3 = Step.new tmpfile.step3 do 
       step1 = dependencies.first
-      stream = step1.get_stream
+      stream = step1.stream
 
       Open.open_pipe do |sin|
         while line = stream.gets
@@ -199,14 +198,14 @@ class TestWorkflowStep < Test::Unit::TestCase
       mutex = Mutex.new
       Open.open_pipe do |sin|
         t2 = Thread.new do
-          stream2 = step2.get_stream
+          stream2 = step2.stream
           while line = stream2.gets
             sin.puts line
           end
         end
 
         t3 = Thread.new do
-          stream3 = step3.get_stream
+          stream3 = step3.stream
           while line = stream3.gets
             sin.puts line
           end
@@ -219,13 +218,14 @@ class TestWorkflowStep < Test::Unit::TestCase
     step4.dependencies = [step2, step3]
 
     lines = []
-    io = step4.run
-    Log::ProgressBar.with_bar do |b|
+    io = step4.run(true)
+    Log::ProgressBar.with_bar severity: 0 do |b|
       while line = io.gets
         b.tick
         lines << line.strip
       end
     end
+    io.close
 
     assert_equal times, lines.length
   end