rbbt-util 5.28.6 → 5.28.11

data/test/rbbt/util/misc/test_bgzf.rb

@@ -2,12 +2,14 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_he
 require 'rbbt/util/misc/bgzf'
 
 class TestBgzf < Test::Unit::TestCase
-  def test_Bgzf
+  def _test_Bgzf
     content = "1234567890" * 1000000
     TmpFile.with_file(content) do |file|
       compressed = file + '.gz'
       `bgzip #{file} -c > #{compressed}`
       stream = Bgzf.setup File.open(compressed)
+      assert_equal "1234", stream.read(4)
+      assert_equal "56", stream.read(2)
       stream.seek 500003
       assert_equal "4567", stream.read(4)
       assert_equal "89", stream.read(2)
@@ -20,23 +22,25 @@ class TestBgzf < Test::Unit::TestCase
       Misc.benchmark do
         tsv = TSV.open(Open.open(file))
       end
-      compressed = file + '.bgz'
 
-      `bgzip #{file} -c > #{compressed}`
-      stream = Bgzf.setup File.open(compressed)
+      `gzip #{file}`
+      stream = Open.open(file + '.gz')
       Misc.benchmark do
         tsv = TSV.open(stream)
       end
 
-      `gzip #{file}`
-      stream = Open.open(file + '.gz')
+      `gunzip #{file}.gz`
+      compressed = file + '.bgz'
+      `bgzip #{file} -c > #{compressed}`
+      stream = Bgzf.setup File.open(compressed)
       Misc.benchmark do
         tsv = TSV.open(stream)
       end
+
     end
   end
 
-  def test_bgzip
+  def _test_bgzip
     assert File.exist?(Bgzf.bgzip_cmd)
     assert 'bgzip', File.basename(Bgzf.bgzip_cmd)
   end

data/test/rbbt/util/misc/test_lock.rb

@@ -48,7 +48,6 @@ if __FILE__ == $0
   TmpFile.with_file do |dir|
     Structure.workdir = dir
     Path.setup dir
-    Log.severity = 4
     TSV.traverse (0..size).to_a, :cpus => cpus, :type => :array, :bar => true do |i|
       begin
         v = rand(num).to_s

data/test/rbbt/util/misc/test_multipart_payload.rb

@@ -185,7 +185,7 @@ END
     inputs[:input2] = "Input2"
     num = 50
     cpus = 1
-    Log.severity = 0
+
     Misc.bootstrap((0..num-1).to_a, cpus) do  |n|
       puts mutipart
       TmpFile.with_file(mutipart, false) do |tmpfile|

data/test/rbbt/util/misc/test_pipes.rb

@@ -131,7 +131,6 @@ row2 AA BB CC
 row3 AAA BBB CCC
 row1 A B C
     EOF
-    Log.severity = 0
 
     text = text * 10000
     TmpFile.with_file(text) do |tmp|
@@ -159,7 +158,6 @@ row2 AA BB CC
 row3 AAA BBB CCC
 row1 A B C
     EOF
-    Log.severity = 0
 
     text = text * 10000
     num = 5
@@ -200,7 +198,6 @@ line3
 line1
     EOF
 
-    Log.severity = 0
     TmpFile.with_file(text1) do |file1|
       TmpFile.with_file(text2) do |file2|
         assert ! Misc.remove_lines(file1, file2, true).read.split("\n").include?("line1")
@@ -224,7 +221,6 @@ line1
 line5
     EOF
 
-    Log.severity = 0
     TmpFile.with_file(text1) do |file1|
       TmpFile.with_file(text2) do |file2|
         assert Misc.select_lines(file1, file2, true).read.split("\n").include?("line1")
@@ -258,7 +254,6 @@ line3
 line4
     EOF
 
-    Log.severity = 0
     TmpFile.with_file(text) do |file|
       io = Open.open(file)
       lines = Set.new

data/test/rbbt/util/test_R.rb

@@ -9,6 +9,7 @@ class TestR < Test::Unit::TestCase
   def test_tsv_R
     tsv = TSV.setup({:a => 1, :b => 2})
     tsv2 = tsv.R <<-EOF
+str(data)
 data = data + 1
     EOF
     assert_equal "2", tsv2["a"].first

data/test/rbbt/util/test_log.rb

@@ -2,7 +2,7 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helpe
 require 'rbbt/util/log'
 
 class TestLog < Test::Unit::TestCase
-  def _test_get_level
+  def test_get_level
     assert_equal 0, Log.get_level(:debug)
     assert_equal 1, Log.get_level(:low)
     assert_equal 1, Log.get_level("LOW")
@@ -10,18 +10,17 @@ class TestLog < Test::Unit::TestCase
     assert_equal 0, Log.get_level(nil)
   end
 
-  def _test_color
+  def test_color
     assert Log.color(:green, "green")
   end
 
-  def _test_no_stderr
+  def test_no_stderr
     Log.ignore_stderr do
       STDERR.puts "NOPRINT"
     end
   end
 
-  def _test_trap_stderr
-    Log.severity = 0
+  def test_trap_stderr
     Log.trap_stderr do
       STDERR.puts "NOPRINT"
       STDERR.puts "NOPRINT"
@@ -40,7 +39,6 @@ class TestLog < Test::Unit::TestCase
   end
 
   def test_trap_std
-    Log.severity = 0
     Log.trap_std do
       STDERR.puts "NOPRINT STDERR"
       STDOUT.puts "NOPRINT STDOUT"

data/test/rbbt/util/test_misc.rb

@@ -475,7 +475,6 @@ eum fugiat quo voluptas nulla pariatur?"
   end
 
   def test_bootstrap
-    Log.severity = 0
     res = Misc.bootstrap((1..10).to_a, 2, :bar => "Test bootstrap ticks", :respawn => :always, :into => []) do |num|
       sleep 1 + rand(2)
       num
@@ -550,7 +549,6 @@ eum fugiat quo voluptas nulla pariatur?"
   end
 
   def __test_bench_log
-    Log.severity = 1
     Misc.benchmark(1000) do
       Log.info { "Hola" }
     end

data/test/rbbt/util/test_open.rb

@@ -169,7 +169,6 @@ class TestOpen < Test::Unit::TestCase
   end
 
   def test_write_stream_repo
-    Log.severity = 0
     TmpFile.with_file do |tmpdir|
       tmpdir = Rbbt.tmp.repo_dir.find
       repo = File.join(tmpdir, 'repo')

data/test/rbbt/util/test_python.rb

@@ -42,7 +42,6 @@ def python_test(a, b):
   end
 
   def test_run_log
-    Log.severity = 0
     TmpFile.with_file do |tmpdir|
       code =<<-EOF
 import sys
@@ -66,3 +65,20 @@ def python_print():
     end
   end
 
+  def test_keras
+    defined = RbbtPython.run do
+      pyimport "keras.models", as: :km
+      defined?(km.Sequential)
+    end
+    assert defined
+  end
+
+  def test_keras_import
+    defined = RbbtPython.run do
+      pyfrom "keras.models", import: :Sequential
+      defined?(self::Sequential)
+    end
+    assert defined
+  end
+end
+

data/test/rbbt/workflow/test_remote_workflow.rb

@@ -88,7 +88,7 @@ class TestRemoteWorkflow < Test::Unit::TestCase
   end
 
 
-  def _test_ssh
+  def test_ssh
     Log.severity = 0
     client = RemoteWorkflow.new "ssh://#{ENV["HOSTNAME"]}:Translation", "Translation"
     job = client.job("translate", "SSH-TEST-1", :genes => ["TP53","KRAS"])

data/test/rbbt/workflow/test_step.rb

@@ -1,8 +1,10 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/workflow'
 require 'rbbt/workflow/task'
 require 'rbbt/workflow/step'
 require 'rbbt/tsv'
 require 'rbbt'
+require 'rbbt-util'
 
 class TestStep < Test::Unit::TestCase
 
@@ -23,8 +25,9 @@ class TestStep < Test::Unit::TestCase
     str2 = "TEST2"
     TmpFile.with_file do |tmpfile|
 
-      task1  = Task.setup :result_type => nil do 
+      task1  = Task.setup :result_type => :string do 
         Open.write(tmpfile, str); 
+        "done"
       end
       step1 = Step.new tmpfile + 'step1', task1
 
@@ -50,9 +53,10 @@ class TestStep < Test::Unit::TestCase
   def __test_dependency_log_relay
     str = "TEST"
     TmpFile.with_file do |tmpfile|
-      task1  = Task.setup :result_type => nil, :name => :task1 do 
+      task1  = Task.setup :result_type => :string, :name => :task1 do 
         log(:starting_task1, "Starting Task1")
         Open.write(tmpfile, str); 
+        "done"
       end
       step1 = Step.new tmpfile + 'step1', task1
 
@@ -69,9 +73,10 @@ class TestStep < Test::Unit::TestCase
   def test_log_relay_step
     str = "TEST"
     TmpFile.with_file do |tmpfile|
-      task1  = Task.setup :result_type => nil, :name => :task1 do 
+      task1  = Task.setup :result_type => :string, :name => :task1 do 
         log(:starting_task1, "Starting Task1")
         Open.write(tmpfile, str); 
+        "done"
       end
       step1 = Step.new tmpfile + 'step1', task1
 

data/test/rbbt/workflow/util/test_orchestrator.rb

@@ -0,0 +1,273 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/workflow/util/orchestrator'
+require 'rbbt/workflow/util/trace'
+require 'rbbt-util'
+require 'rbbt/workflow'
+
+module TestWF
+  extend Workflow 
+  
+  MULT = 0.1
+  task :a => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+  
+  dep :a
+  task :b => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+  
+  dep :a
+  dep :b
+  task :c => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+  
+  dep :c
+  task :d => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+
+  dep :c
+  task :e => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
+end
+
+class TestClass < Test::Unit::TestCase
+  def test_orchestrate_resources
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 0 do
+      orchestrator.process(rules, jobs)
+    end
+
+    data = Workflow.trace jobs, :plot_data => true
+    eend = data.column("End.second").values.collect{|v| v.to_f}.max
+    second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
+    (0..eend.to_i).each do |second|
+      tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
+      cpus = 0
+      tasks.through :key, ["Workflow", "Task"] do |k, values|
+        workflow, task = values
+        cpus += rules[workflow][task.to_s]["resources"]["cpus"]
+      end
+      second_cpus[second] = cpus
+    end
+
+    assert Misc.mean(second_cpus.values) > 15
+    assert Misc.mean(second_cpus.values) < 30
+  end
+
+  def test_orchestrate_erase
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    erase: true
+    resources:
+      cpus: 7
+  b:
+    erase: true
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def test_orchestrate_default
+
+    jobs =[]
+
+    num = 3
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    erase: true
+    resources:
+      cpus: 7
+  b:
+    erase: true
+    resources:
+      cpus: 2
+  c:
+    erase: false
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def test_orchestrate_top_level
+
+    jobs =[]
+
+    num = 3
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:c, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd'
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def test_orchestrate_top_level_double_dep
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:e, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd' || job.task_name.to_s == 'e'
+      assert job.info[:archived_info].keys.select{|k| k.include?("TestWF/c/")}.any?
+      assert job.info[:archived_info].keys.select{|k| k.include?("TestWF/c/")}.any?
+    end
+
+  end
+end
+