scout-gear 8.0.0 → 8.1.0

data/test/scout/test_work_queue.rb

@@ -90,60 +90,82 @@ class TestWorkQueue < Test::Unit::TestCase
   end
 
   def test_queue_error
-    num = 100
-    reps = 10_000
-
-    q = WorkQueue.new num do |obj|
-      raise ScoutException if rand < 0.1
-      [Process.pid.to_s, obj.to_s] * " "
-    end
-
-    res = []
-    q.process do |out|
-      res << out
-    end
+    5.times do |i|
+      num = 100
+      reps = 10_000
 
-    Log.with_severity 7 do
-      pid = Process.fork do
-        reps.times do |i|
-          q.write i
-        end
+      q = WorkQueue.new num do |obj|
+        raise ScoutException if rand < 0.1
+        [Process.pid.to_s, obj.to_s] * " "
       end
 
-      Process.waitpid pid
+      res = []
+      q.process do |out|
+        res << out
+      end
 
-      assert_raise ScoutException do
-        q.join
-        t.join
+      Log.with_severity 7 do
+        t = Thread.new do
+          Thread.current["name"] = "queue writer"
+          Thread.current.report_on_exception = false
+          reps.times do |i|
+            q.write i
+          end
+          q.close
+        end
+        Thread.pass until t["name"]
+
+        assert_raise ScoutException do
+          begin
+            q.join(false)
+          rescue
+            t.raise($!)
+            raise $!
+          ensure
+            t.join
+          end
+        end
       end
     end
   end
 
   def test_queue_error_in_input
-    num = 100
-    reps = 10_000
+    5.times do |i|
+      num = 100
+      reps = 10_000
 
-    q = WorkQueue.new num do |obj|
-      [Process.pid.to_s, obj.to_s] * " "
-    end
-
-    res = []
-    q.process do |out|
-      raise ScoutException 
-      res << out
-    end
+      q = WorkQueue.new num do |obj|
+        [Process.pid.to_s, obj.to_s] * " "
+      end
 
-    Log.with_severity 7 do
-      pid = Process.fork do
-        reps.times do |i|
-          q.write i
-        end
-        q.close
+      res = []
+      q.process do |out|
+        raise ScoutException 
+        res << out
       end
 
-      assert_raise ScoutException do
-        q.join
-        t.join
+      Log.with_severity 7 do
+        t = Thread.new do
+          Thread.current.report_on_exception = false
+          Thread.current["name"] = "queue writer"
+          reps.times do |i|
+            q.write i
+          end
+          q.close
+        end
+        Thread.pass until t["name"]
+
+        assert_raise ScoutException do
+          begin
+            q.join(false)
+          rescue Exception
+            t.raise($!)
+            raise $!
+          ensure
+            t.join
+            q.clean
+          end
+        end
       end
     end
   end

data/test/scout/tsv/persist/test_adapter.rb

@@ -3,7 +3,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
 
 require 'scout/tsv'
 class TestTSVAdapter < Test::Unit::TestCase
-  def _test_get_set
+  def test_get_set
     tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
     tsv.type = :list
     tsv.extend TSVAdapter

data/test/scout/tsv/test_index.rb

@@ -43,6 +43,20 @@ row2    a    b    id3
     end
   end
 
+  def test_index_from_flat
+    content =<<-'EOF'
+#: :sep=" "#:type=:flat
+#Id    ValueA
+row1    a aa aaa
+row2    b bb bbb
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      index = TSV.index(filename, :target => "Id")
+      assert_equal "row1", index["aa"]
+    end
+  end
+
   def test_from_tsv
     content =<<-'EOF'
 #: :sep=/\s+/#:type=:double#:merge=:concat

data/test/scout/tsv/test_parser.rb

@@ -119,6 +119,20 @@ k a|A b|B
     assert_equal [%w(b B), %w(k)], tsv['k']
   end
 
+  def test_parse_flat
+    content =<<-EOF
+#: :sep=" "#:type=:flat
+#Key ValueA
+row1 a aa aaa
+row2 b bb bbb
+    EOF
+
+    tsv = TSV.open(content)
+    assert_equal %w(a aa aaa), tsv["row1"]
+    tsv = TSV.open(content, :fields => ["ValueA"])
+    assert_equal %w(a aa aaa), tsv["row1"]
+  end
+
   def test_parse_key
     content =<<-EOF
 #: :sep=" "#:type=:double

data/test/scout/tsv/test_stream.rb

@@ -0,0 +1,200 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'scout/tsv'
+class TestTSVStream < Test::Unit::TestCase
+  def test_paste_stream
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row1 A B C
+row2 AA BB CC
+row3 AAA BBB CCC
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labela Labelb 
+row1 a b
+row2 aa bb
+row3 aaa bbb
+    EOF
+
+    text3=<<-EOF
+#: :sep=" "
+#Row LabelC
+row1 c
+row2 cc
+row3 ccc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list)
+    assert_equal ["A", "B", "C", "a", "b", "c"], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["AAA", "BBB", "CCC", "aaa", "bbb", "ccc"], tsv["row3"]
+  end
+
+  def test_paste_stream_sort
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row1 A B C
+row3 AAA BBB CCC
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labela Labelb
+row1 a b
+row3 aaa bbb
+row2 aa bb
+    EOF
+
+    text3=<<-EOF
+#: :sep=" "
+#Row Labelc
+row3 ccc
+row1 c
+row2 cc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list, :sort => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal %w(LabelA LabelB LabelC Labela Labelb Labelc), tsv.fields
+    assert_equal ["A", "B", "C", "a", "b", "c"], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["AAA", "BBB", "CCC", "aaa", "bbb", "ccc"], tsv["row3"]
+  end
+
+  def test_paste_stream_missing_2
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row1 A B C
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labela Labelb
+row2 aa bb
+    EOF
+
+    text3=<<-EOF
+#: :sep=" "
+#Row Labelc
+row3 ccc
+row2 cc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list, :sort => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal %w(LabelA LabelB LabelC Labela Labelb Labelc), tsv.fields
+    assert_equal ["A", "B", "C", "", "", ""], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["", "", "", "", "", "ccc"], tsv["row3"]
+  end
+
+  def test_paste_stream_missing
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row1 A B C
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labela Labelb
+row2 aa bb
+    EOF
+
+    text3=<<-EOF
+#: :sep=" "
+#Row Labelc
+row3 ccc
+row2 cc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    s3 = StringIO.new text3
+    tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list, :sort => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal %w(LabelA LabelB LabelC Labela Labelb Labelc), tsv.fields
+    assert_equal ["A", "B", "C", "", "", ""], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
+    assert_equal ["", "", "", "", "", "ccc"], tsv["row3"]
+  end
+
+  def test_paste_stream_missing_3
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row1 A B C
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row Labelc
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    tsv = TSV.open TSV.paste_streams([s1,s2], :sep => " ", :type => :list, :sort => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal %w(LabelA LabelB LabelC Labelc), tsv.fields
+    assert_equal ["A", "B", "C", ""], tsv["row1"]
+    assert_equal ["AA", "BB", "CC", ""], tsv["row2"]
+  end
+
+  def test_paste_stream_same_field
+    text1=<<-EOF
+#: :sep=" "
+#Row LabelA
+row1 A
+row2 AA
+    EOF
+
+    text2=<<-EOF
+#: :sep=" "
+#Row LabelA
+row2 AAA
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    tsv = TSV.open TSV.paste_streams([s1,s2], :sep => " ", :type => :double, :sort => false, :same_fields => true)
+    assert_equal "Row", tsv.key_field
+    assert_equal ["AA", "AAA"], tsv["row2"][0]
+  end
+
+  def test_paste_stream_nohead
+    text1=<<-EOF
+row1\tA
+row2\tAA
+    EOF
+
+    text2=<<-EOF
+row2\tAAA
+    EOF
+
+    s1 = StringIO.new text1
+    s2 = StringIO.new text2
+    tsv = TSV.open TSV.paste_streams([s1,s2], :type => :double, :sort => false, :same_fields => true)
+    assert_equal ["AA", "AAA"], tsv["row2"][0]
+  end
+
+end
+

data/test/scout/tsv/test_transformer.rb

@@ -105,4 +105,16 @@ row2    A2|A22    B2|B22
     assert_equal "A1", tsv.to_single["row1"]
     assert_equal "A2", tsv.to_single["row2"]
   end
+
+  def test_to_flat
+    content =<<-EOF
+#: :sep=" "
+#ID    ValueA    ValueB
+row1    A1|A11    B1|B11
+row2    A2|A22    B2|B22
+    EOF
+
+    tsv = TSV.open(content)
+    assert_equal %w(A1 A11 B1 B11), tsv.to_flat["row1"]
+  end
 end

data/test/scout/workflow/deployment/test_orchestrator.rb

@@ -0,0 +1,272 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+class TestOrchestrator < Test::Unit::TestCase
+  setup do
+    module TestWF
+      extend Workflow 
+
+      MULT = 0.01
+      task :a => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :a
+      task :b => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :a
+      dep :b
+      task :c => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :c
+      task :d => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+
+      dep :c
+      task :e => :text do
+        sleep(TestWF::MULT * (rand(10) + 2))
+      end
+    end
+  end
+
+  def TODO_test_orchestrate_resources
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 0 do
+      orchestrator.process(rules, jobs)
+    end
+
+    data = Workflow.trace jobs, :plot_data => true
+    eend = data.column("End.second").values.collect{|v| v.to_f}.max
+    second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
+    (0..eend.to_i).each do |second|
+      tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
+      cpus = 0
+      tasks.through :key, ["Workflow", "Task"] do |k, values|
+        workflow, task = values
+        cpus += rules[workflow][task.to_s]["resources"]["cpus"]
+      end
+      second_cpus[second] = cpus
+    end
+
+    assert Misc.mean(second_cpus.values) > 15
+    assert Misc.mean(second_cpus.values) < 30
+  end
+
+  def TODO_test_orchestrate_erase
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    erase: true
+    resources:
+      cpus: 7
+  b:
+    erase: true
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def TODO_test_orchestrate_default
+
+    jobs =[]
+
+    num = 3
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    erase: true
+    resources:
+      cpus: 7
+  b:
+    erase: true
+    resources:
+      cpus: 2
+  c:
+    erase: false
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def TODO_test_orchestrate_top_level
+
+    jobs =[]
+
+    num = 3
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:c, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd'
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def TODO_test_orchestrate_top_level_double_dep
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:e, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd' || job.task_name.to_s == 'e'
+      assert job.info[:archived_info].keys.select{|k| k.include?("TestWF/c/")}.any?
+      assert job.info[:archived_info].keys.select{|k| k.include?("TestWF/c/")}.any?
+    end
+
+  end
+end
+