rbbt-util 5.28.7 → 5.28.12

data/test/rbbt/util/test_open.rb

@@ -169,7 +169,6 @@ class TestOpen < Test::Unit::TestCase
   end
 
   def test_write_stream_repo
-    Log.severity = 0
     TmpFile.with_file do |tmpdir|
       tmpdir = Rbbt.tmp.repo_dir.find
       repo = File.join(tmpdir, 'repo')

data/test/rbbt/util/test_python.rb

@@ -42,7 +42,6 @@ def python_test(a, b):
   end
 
   def test_run_log
-    Log.severity = 0
     TmpFile.with_file do |tmpdir|
       code =<<-EOF
 import sys
@@ -66,3 +65,20 @@ def python_print():
     end
   end
 
+  def test_keras
+    defined = RbbtPython.run do
+      pyimport "keras.models", as: :km
+      defined?(km.Sequential)
+    end
+    assert defined
+  end
+
+  def test_keras_import
+    defined = RbbtPython.run do
+      pyfrom "keras.models", import: :Sequential
+      defined?(self::Sequential)
+    end
+    assert defined
+  end
+end
+

data/test/rbbt/workflow/test_remote_workflow.rb

@@ -88,7 +88,7 @@ class TestRemoteWorkflow < Test::Unit::TestCase
   end
 
 
-  def _test_ssh
+  def test_ssh
     Log.severity = 0
     client = RemoteWorkflow.new "ssh://#{ENV["HOSTNAME"]}:Translation", "Translation"
     job = client.job("translate", "SSH-TEST-1", :genes => ["TP53","KRAS"])

data/test/rbbt/workflow/test_step.rb

@@ -1,8 +1,10 @@
 require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/workflow'
 require 'rbbt/workflow/task'
 require 'rbbt/workflow/step'
 require 'rbbt/tsv'
 require 'rbbt'
+require 'rbbt-util'
 
 class TestStep < Test::Unit::TestCase
 
@@ -23,8 +25,9 @@ class TestStep < Test::Unit::TestCase
     str2 = "TEST2"
     TmpFile.with_file do |tmpfile|
 
-      task1  = Task.setup :result_type => nil do 
+      task1  = Task.setup :result_type => :string do 
         Open.write(tmpfile, str); 
+        "done"
       end
       step1 = Step.new tmpfile + 'step1', task1
 
@@ -50,9 +53,10 @@ class TestStep < Test::Unit::TestCase
   def __test_dependency_log_relay
     str = "TEST"
     TmpFile.with_file do |tmpfile|
-      task1  = Task.setup :result_type => nil, :name => :task1 do 
+      task1  = Task.setup :result_type => :string, :name => :task1 do 
         log(:starting_task1, "Starting Task1")
         Open.write(tmpfile, str); 
+        "done"
       end
       step1 = Step.new tmpfile + 'step1', task1
 
@@ -69,9 +73,10 @@ class TestStep < Test::Unit::TestCase
   def test_log_relay_step
     str = "TEST"
     TmpFile.with_file do |tmpfile|
-      task1  = Task.setup :result_type => nil, :name => :task1 do 
+      task1  = Task.setup :result_type => :string, :name => :task1 do 
         log(:starting_task1, "Starting Task1")
         Open.write(tmpfile, str); 
+        "done"
       end
       step1 = Step.new tmpfile + 'step1', task1
 

data/test/rbbt/workflow/util/test_orchestrator.rb

@@ -17,6 +17,7 @@ module TestWF
     sleep(TestWF::MULT * (rand(10) + 2))
   end
   
+  dep :a
   dep :b
   task :c => :text do
     sleep(TestWF::MULT * (rand(10) + 2))
@@ -26,16 +27,21 @@ module TestWF
   task :d => :text do
     sleep(TestWF::MULT * (rand(10) + 2))
   end
+
+  dep :c
+  task :e => :text do
+    sleep(TestWF::MULT * (rand(10) + 2))
+  end
 end
 
 class TestClass < Test::Unit::TestCase
-  def _test_orchestrate
+  def test_orchestrate_resources
 
     jobs =[]
 
     num = 10
     num.times do |i|
-      jobs.concat %w(test1 _test2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
     end
     jobs.each do |j| j.recursive_clean end
 
@@ -81,13 +87,13 @@ TestWF:
     assert Misc.mean(second_cpus.values) < 30
   end
 
-  def test_orchestrate_size
+  def test_orchestrate_erase
 
     jobs =[]
 
     num = 10
     num.times do |i|
-      jobs.concat %w(test1 _test2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
     end
     jobs.each do |j| j.recursive_clean end
 
@@ -98,9 +104,11 @@ default_resources:
   IO: 1
 TestWF:
   a:
+    erase: true
     resources:
       cpus: 7
   b:
+    erase: true
     resources:
       cpus: 2
   c:
@@ -112,25 +120,154 @@ TestWF:
     EOF
 
     orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
-    Log.with_severity 0 do
+    Log.with_severity 3 do
       orchestrator.process(rules, jobs)
     end
 
-    data = Workflow.trace jobs, :plot_data => true
-    eend = data.column("End.second").values.collect{|v| v.to_f}.max
-    second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
-    (0..eend.to_i).each do |second|
-      tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
-      cpus = 0
-      tasks.through :key, ["Workflow", "Task"] do |k, values|
-        workflow, task = values
-        cpus += rules[workflow][task.to_s]["resources"]["cpus"]
-      end
-      second_cpus[second] = cpus
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def test_orchestrate_default
+
+    jobs =[]
+
+    num = 3
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    erase: true
+    resources:
+      cpus: 7
+  b:
+    erase: true
+    resources:
+      cpus: 2
+  c:
+    erase: false
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def test_orchestrate_top_level
+
+    jobs =[]
+
+    num = 3
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:c, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd'
+      assert job.step(:c).dependencies.empty?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
+      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
+    end
+
+  end
+
+  def test_orchestrate_top_level_double_dep
+
+    jobs =[]
+
+    num = 10
+    num.times do |i|
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:e, name + " #{i}") }
+      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
+    end
+    jobs.each do |j| j.recursive_clean end
+
+    rules = YAML.load <<-EOF
+defaults:
+  erase: true
+  log: 4
+default_resources:
+  IO: 1
+TestWF:
+  a:
+    resources:
+      cpus: 7
+  b:
+    resources:
+      cpus: 2
+  c:
+    resources:
+      cpus: 10
+  d:
+    resources:
+      cpus: 15
+    EOF
+
+    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
+    Log.with_severity 3 do
+      orchestrator.process(rules, jobs)
+    end
+
+    jobs.each do |job|
+      next unless job.task_name.to_s == 'd' || job.task_name.to_s == 'e'
+      assert job.info[:archived_info].keys.select{|k| k.include?("TestWF/c/")}.any?
+      assert job.info[:archived_info].keys.select{|k| k.include?("TestWF/c/")}.any?
     end
 
-    assert Misc.mean(second_cpus.values) > 15
-    assert Misc.mean(second_cpus.values) < 30
   end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.28.7
+  version: 5.28.12
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-11-02 00:00:00.000000000 Z
+date: 2020-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -318,6 +318,8 @@ files:
 - lib/rbbt/workflow/definition.rb
 - lib/rbbt/workflow/doc.rb
 - lib/rbbt/workflow/examples.rb
+- lib/rbbt/workflow/integration/ansible.rb
+- lib/rbbt/workflow/integration/ansible/workflow.rb
 - lib/rbbt/workflow/integration/cromwell.rb
 - lib/rbbt/workflow/integration/nextflow.rb
 - lib/rbbt/workflow/remote_workflow.rb
@@ -347,6 +349,7 @@ files:
 - share/config.ru
 - share/install/software/lib/install_helpers
 - share/rbbt_commands/alias
+- share/rbbt_commands/ansible
 - share/rbbt_commands/app/install
 - share/rbbt_commands/app/start
 - share/rbbt_commands/app/template
@@ -513,7 +516,6 @@ files:
 - test/rbbt/util/test_simpleDSL.rb
 - test/rbbt/util/test_simpleopt.rb
 - test/rbbt/util/test_tmpfile.rb
-- test/rbbt/workflow/remote/test_client.rb
 - test/rbbt/workflow/step/test_dependencies.rb
 - test/rbbt/workflow/test_doc.rb
 - test/rbbt/workflow/test_remote_workflow.rb
@@ -552,7 +554,6 @@ test_files:
 - test/rbbt/workflow/test_doc.rb
 - test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
-- test/rbbt/workflow/remote/test_client.rb
 - test/rbbt/workflow/step/test_dependencies.rb
 - test/rbbt/workflow/test_task.rb
 - test/rbbt/resource/test_path.rb

data/test/rbbt/workflow/remote/test_client.rb

@@ -1,56 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/workflow'
-require 'rbbt/workflow/remote/client'
-
-class TestWFRest
-  extend Workflow
-
-  input :name, :string, "Name to greet", "World"
-  task :hi => :string do |name|
-    "Hello #{name}"
-  end
-
-  dep :hi
-  task :intro => :string do 
-    step(:hi).load + ", nice to meet you"
-  end
-end
-
-class TestRemote < Test::Unit::TestCase
-
-  def test_ssh
-    Log.severity = 0
-    client = WorkflowRemoteClient.new "ssh://turbo:Translation", "Translation"
-    job = client.job("translate", "SSH-TEST", :genes => ["TP53","KRAS"])
-    assert_equal 2, job.run.select{|l| l =~ /ENSG/}.length
-  end
-
-  def test_rest
-    Log.with_severity 0 do
-
-      workflow_server(TestWFRest) do |client|
-        assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
-        assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
-        assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
-      end
-
-      workflow_server(TestWFRest, :Port => 1902) do |client|
-        assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
-        assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
-        assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
-      end
-    end
-  end
-
-  def test_rest_clean
-    real_job = TestWFRest.job(:hi)
-    real_job.run
-    first_time = File.ctime(real_job.path)
-    workflow_server(TestWFRest) do |client|
-      assert File.ctime(real_job.path) == first_time
-      assert_equal "Hello World", client.job(:hi, nil, {}).clean.run
-      assert File.ctime(real_job.path) > first_time
-    end
-  end
-end
-