rbbt-util 5.42.0 → 5.44.0

data/test/rbbt/workflow/test_step.rb

@@ -1,231 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
-require 'rbbt/workflow'
-require 'rbbt/workflow/task'
-require 'rbbt/workflow/step'
-require 'rbbt/tsv'
-require 'rbbt'
-require 'rbbt-util'
-
-class TestStep < Test::Unit::TestCase
-
-  def test_step
-    task  = Task.setup do "TEST" end
-    task2  = Task.setup do raise "Persistence ignored" end
-    TmpFile.with_file do |tmp|
-      step = Step.new tmp, task
-      assert_equal "TEST", step.run
-      assert File.exist? tmp
-      step = Step.new tmp, task2
-      assert_equal "TEST", step.run
-    end
-  end
-
-  def test_dependency
-    str = "TEST"
-    str2 = "TEST2"
-    TmpFile.with_file do |tmpfile|
-
-      task1  = Task.setup :result_type => :string do 
-        Open.write(tmpfile, str); 
-        "done"
-      end
-      step1 = Step.new tmpfile + 'step1', task1
-
-      task2  = Task.setup :result_type => :string do 
-        Open.read(tmpfile) 
-      end
-      step2 = Step.new tmpfile + 'step2', task2, [], [step1]
-
-      step2.run
-      assert_equal "TEST", Open.read(tmpfile + 'step2')
-
-      task2  = Task.setup :result_type => :string do 
-        str2 
-      end
-      step2 = Step.new tmpfile + 'step2', task2, [], step1
-
-      step1.clean
-      step2.clean.run
-      assert_equal "TEST2", Open.read(tmpfile + 'step2')
-    end
-  end
-
-  def __test_dependency_log_relay
-    str = "TEST"
-    TmpFile.with_file do |tmpfile|
-      task1  = Task.setup :result_type => :string, :name => :task1 do 
-        log(:starting_task1, "Starting Task1")
-        Open.write(tmpfile, str); 
-        "done"
-      end
-      step1 = Step.new tmpfile + 'step1', task1
-
-      task2  = Task.setup :result_type => :string, :name => :task1 do 
-        Open.read(tmpfile) 
-      end
-      step2 = Step.new tmpfile + 'step2', task2, [], [step1]
-
-      step2.run
-      assert step2.messages.include? "Starting Task1"
-    end
-  end
-
-  def test_log_relay_step
-    str = "TEST"
-    TmpFile.with_file do |tmpfile|
-      task1  = Task.setup :result_type => :string, :name => :task1 do 
-        log(:starting_task1, "Starting Task1")
-        Open.write(tmpfile, str); 
-        "done"
-      end
-      step1 = Step.new tmpfile + 'step1', task1
-
-
-      task2  = Task.setup :result_type => :string, :name => :task1 do 
-        Open.read(tmpfile) 
-      end
-      step2 = Step.new tmpfile + 'step2', task2, [], [step1]
-
-      Step.log_relay_step = step2
-      step2.run
-
-      assert step2.messages.include? "Starting Task1"
-    end
-  end
-
-
-  def test_exec
-    TmpFile.with_file do |lock|
-      task  = Task.setup do "TEST" end
-      TmpFile.with_file do |tmp|
-        step = Step.new tmp, task
-        assert_equal "TEST", step.exec
-      end
-    end
-  end
-
-
-  def __test_fork
-    TmpFile.with_file do |lock|
-      task  = Task.setup do while not File.exist?(lock) do sleep 1; end; "TEST" end
-      TmpFile.with_file do |tmp|
-        step = Step.new tmp, task
-        job = step.fork
-        assert !job.done?
-        assert_raise RuntimeError do step.fork end
-        sleep 1
-        Open.write(lock, "open")
-        assert_equal "TEST", job.join.load
-        assert job.done?
-      end
-    end
-  end
-
-  def __test_abort
-    TmpFile.with_file do |lock|
-      task  = Task.setup do while not File.exist?(lock) do sleep 1; end; "TEST" end
-      TmpFile.with_file do |tmp|
-        step = Step.new tmp, task
-        job = step.fork
-        assert !job.done?
-        step.clean.fork 
-        job.abort 
-        assert_equal :aborted, job.status
-        Open.write(lock, "open")
-        job.clean.fork 
-        job.join
-        assert job.done?
-      end
-    end
-  end
-
-  def test_files
-    TmpFile.with_file do |lock|
-      task  = Task.setup do 
-        Open.write(file("test"),"TEST")
-      end
-      TmpFile.with_file do |tmp|
-        step = Step.new tmp, task
-        #job = step.fork
-        #while not job.done? do sleep 1 end
-        step.run
-        assert_equal "TEST", Open.read(step.file("test"))
-      end
-    end
-  end
-
-  def test_messages
-    TmpFile.with_file do |lock|
-
-      task  = Task.setup do 
-        message "WRITE"
-        Open.write(file("test"),"TEST")
-        Open.write(path,"done")
-        nil
-      end
-
-      TmpFile.with_file do |tmp|
-        step = Step.new tmp, task
-        job = step
-        step.run
-        while not job.done? do sleep 1 end
-        assert_equal "TEST", Open.read(job.file("test"))
-        assert job.messages.include? "WRITE"
-      end
-    end
-  end
-
-  def test_subdir
-    TmpFile.with_file do |lock|
-      task  = Task.setup do 
-        message "WRITE"
-        Open.write(file("test"),"TEST")
-      end
-
-      TmpFile.with_file do |tmp|
-        step = Step.new File.join(tmp, 'subdir1', 'subdir2'), task
-        #job = step.fork
-        #while not job.done? do sleep 1 end
-        step.run
-        assert_equal "TEST", Open.read(step.file("test"))
-        assert step.messages.include? "WRITE"
-      end
-    end
-  end
-
-  def test_semaphore
-    TmpFile.with_file do |semaphore|
-      begin
-        semaphore = "/" << semaphore.gsub('/','_')
-        RbbtSemaphore.create_semaphore(semaphore, 2)
-
-        task  = Task.setup do 
-          5.times do
-            puts "Process: #{Process.pid}"
-            sleep rand 
-          end
-        end
-
-        jobs = []
-        10.times do
-          TmpFile.with_file do |tmp|
-            step = Step.new tmp, task
-            jobs << step.fork(semaphore)
-          end
-        end
-        Step.wait_for_jobs(jobs)
-      ensure
-        RbbtSemaphore.delete_semaphore(semaphore)
-      end
-    end
-  end
-
-  def __test_load_return_description
-    require 'rbbt/workflow'
-    Workflow.require_workflow "Study"
-    study = Study.setup("LICA-FR")
-    job = study.recurrent_mutations(:job)
-    iii job.load.organism
-    iii study.recurrent_mutations.organism
-  end
-end

data/test/rbbt/workflow/test_task.rb

@@ -1,85 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
-require 'rbbt/workflow/task'
-require 'rbbt/tsv'
-require 'rbbt'
-
-class TEST
-  def self.times(str, times)
-    [str] * times 
-  end
-end
-
-class TestTask < Test::Unit::TestCase
-  def test_task_setup
-    a = [1,2]
-    p = Task.setup{|pos| self[pos]}
-
-    assert_equal 1, p.exec_in(a, 0)
-    assert_equal 2, p.exec_in(a, 1)
-  end
-
-  def test_task_exec
-    a = [1,2]
-    p = Proc.new{|pos| self[pos]}
-    p.extend Task
-    assert_equal 1, p.exec_in(a, 0)
-    assert_equal 2, p.exec_in(a, 1)
-  end
-
-  def test_task_named
-    t = Task.setup :inputs => %w(str times) do |str, times| [str] * times end
-    assert_equal %w(test test), t.exec(:str => "test", :times => 2)
-    assert_equal %w(test test), t.exec(:str => "test", :times => 2)
-  end
-
-  def test_task_persist
-    t = Task.setup :inputs => %w(str times) do |str, times| 
-      [str] * times 
-    end
-
-    TmpFile.with_file do |perfile|
-      assert_equal %w(test test), t.persist_exec_in(perfile, Object.new, :str => "test", :times => 2)
-      assert File.exist? perfile
-
-      t = Task.setup :inputs => %w(str times) do |str, times| 
-        raise "Persistence ignored"
-      end
-
-      assert_equal %w(test test), t.persist_exec_in(perfile, Object.new, :str => "test", :times => 2)
-    end
-  end
-
-  def test_task_unbound_method
-    t = TEST.method :times
-    t.extend Task
-    t.inputs = %w(str times) 
-
-    TmpFile.with_file do |perfile|
-      assert_equal %w(test test), t.persist_exec_in(perfile, Object.new, :str => "test", :times => 2)
-      assert File.exist? perfile
-
-      t = Task.setup :inputs => %w(str times) do |str, times| 
-        raise "Persistence ignored"
-      end
-
-      assert_equal %w(test test), t.persist_exec_in(perfile, Object.new, :str => "test", :times => 2)
-    end
-  end
-
-  def test_task_unbound_method2
-    t = Task.setup :inputs => %w(str times), &TEST.method(:times)
-
-    TmpFile.with_file do |perfile|
-      assert_equal %w(test test), t.persist_exec_in(perfile, Object.new, :str => "test", :times => 2)
-      assert File.exist? perfile
-
-      t = Task.setup :inputs => %w(str times) do |str, times| 
-        raise "Persistence ignored"
-      end
-
-      assert_equal %w(test test), t.persist_exec_in(perfile, Object.new, :str => "test", :times => 2)
-    end
-  end
-
-
-end

data/test/rbbt/workflow/util/test_archive.rb

@@ -1,31 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt-util'
-require 'rbbt/workflow'
-require 'rbbt/workflow/util/archive'
-
-module ArchiveTestWF
-  extend Workflow
-  task :test_archive => :string do
-    Open.write(file(:file1), "Test file")
-    "TEST"
-  end
-end
-
-class TestClass < Test::Unit::TestCase
-  def test_migrate
-    job = ArchiveTestWF.job(:test_archive)
-    job.run
-
-    Log.with_severity 0 do
-      TmpFile.with_file do |tmpdir|
-        Misc.in_dir tmpdir do
-          Step.migrate(job.path, :current, :delete => false, :print => false)
-        end
-        assert_equal "TEST", Open.read(File.join(tmpdir, 'var/jobs/ArchiveTestWF/test_archive/Default'))
-      end
-    end
-
-
-  end
-end
-

data/test/rbbt/workflow/util/test_data.rb

@@ -1,67 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/workflow/util/data'
-
-module TestDataWF2
-  extend Workflow
-  extend Workflow::Data
-
-  input :salutation, :string
-  task :salute_antonio => :string do |name,salutation|
-    "Hi Antonio: #{salutation}"
-  end
-end
-
-module TestDataWF
-  extend Workflow
-  extend Workflow::Data
-
-  input :salutation, :string
-  task :salute_luis => :string do |name,salutation|
-    "Hi Luis: #{salutation}"
-  end
-
-  input :name, :string
-  input :salutation, :string
-  task :salute => :string do |name,salutation|
-    "Hi #{name}: #{salutation}"
-  end
-
-  data_task :salute_data, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
-    options.merge({:salutation => directory.salutation.read})
-  end
-
-  data_task :salute_data2, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
-    {:task => :salute_luis, :inputs => options.merge({:salutation => directory.salutation.read})}
-  end
-
-  data_task :salute_data3, TestDataWF, :salute, :salutation => :placeholder do |directory,options|
-    {:task => :salute_antonio, :workflow => TestDataWF2, :inputs => options.merge({:salutation => directory.salutation.read})}
-  end
-
-end
-
-class TestWorkflowData < Test::Unit::TestCase
-  def test_workflow_data
-    TmpFile.with_file do |tmpdir|
-      tmpdir = Path.setup(tmpdir.dup)
-
-      Open.write(tmpdir.TestDir.options.name, "Miguel")
-      Open.write(tmpdir.TestDir.salutation, "My salutations")
-
-      TestDataWF.data_dir tmpdir
-
-      job = TestDataWF.job(:salute_data, "TestDir")
-      job.recursive_clean.run
-      assert job.run.include? "Miguel"
-
-      job = TestDataWF.job(:salute_data2, "TestDir")
-      puts job.recursive_clean.run
-      assert job.run.include? "Luis"
-
-      job = TestDataWF.job(:salute_data3, "TestDir")
-      job.recursive_clean.run
-      assert job.run.include? "Antonio"
-    end
-  end
-end
-

data/test/rbbt/workflow/util/test_orchestrator.rb

@@ -1,273 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/workflow/util/orchestrator'
-require 'rbbt/workflow/util/trace'
-require 'rbbt-util'
-require 'rbbt/workflow'
-
-module TestWF
-  extend Workflow 
-  
-  MULT = 0.1
-  task :a => :text do
-    sleep(TestWF::MULT * (rand(10) + 2))
-  end
-  
-  dep :a
-  task :b => :text do
-    sleep(TestWF::MULT * (rand(10) + 2))
-  end
-  
-  dep :a
-  dep :b
-  task :c => :text do
-    sleep(TestWF::MULT * (rand(10) + 2))
-  end
-  
-  dep :c
-  task :d => :text do
-    sleep(TestWF::MULT * (rand(10) + 2))
-  end
-
-  dep :c
-  task :e => :text do
-    sleep(TestWF::MULT * (rand(10) + 2))
-  end
-end
-
-class TestClass < Test::Unit::TestCase
-  def test_orchestrate_resources
-
-    jobs =[]
-
-    num = 10
-    num.times do |i|
-      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
-    end
-    jobs.each do |j| j.recursive_clean end
-
-    rules = YAML.load <<-EOF
-defaults:
-  log: 4
-default_resources:
-  IO: 1
-TestWF:
-  a:
-    resources:
-      cpus: 7
-  b:
-    resources:
-      cpus: 2
-  c:
-    resources:
-      cpus: 10
-  d:
-    resources:
-      cpus: 15
-    EOF
-
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
-    Log.with_severity 0 do
-      orchestrator.process(rules, jobs)
-    end
-
-    data = Workflow.trace jobs, :plot_data => true
-    eend = data.column("End.second").values.collect{|v| v.to_f}.max
-    second_cpus = TSV.setup({}, "Second~CPUS#:type=:single#:cast=:to_f")
-    (0..eend.to_i).each do |second|
-      tasks = data.select("Start.second"){|s| s <= second}.select("End.second"){|s| s > second}
-      cpus = 0
-      tasks.through :key, ["Workflow", "Task"] do |k, values|
-        workflow, task = values
-        cpus += rules[workflow][task.to_s]["resources"]["cpus"]
-      end
-      second_cpus[second] = cpus
-    end
-
-    assert Misc.mean(second_cpus.values) > 15
-    assert Misc.mean(second_cpus.values) < 30
-  end
-
-  def test_orchestrate_erase
-
-    jobs =[]
-
-    num = 10
-    num.times do |i|
-      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
-    end
-    jobs.each do |j| j.recursive_clean end
-
-    rules = YAML.load <<-EOF
-defaults:
-  log: 4
-default_resources:
-  IO: 1
-TestWF:
-  a:
-    erase: true
-    resources:
-      cpus: 7
-  b:
-    erase: true
-    resources:
-      cpus: 2
-  c:
-    resources:
-      cpus: 10
-  d:
-    resources:
-      cpus: 15
-    EOF
-
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
-    Log.with_severity 3 do
-      orchestrator.process(rules, jobs)
-    end
-
-    jobs.each do |job|
-      assert job.step(:c).dependencies.empty?
-      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
-      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
-    end
-
-  end
-
-  def test_orchestrate_default
-
-    jobs =[]
-
-    num = 3
-    num.times do |i|
-      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
-    end
-    jobs.each do |j| j.recursive_clean end
-
-    rules = YAML.load <<-EOF
-defaults:
-  erase: true
-  log: 4
-default_resources:
-  IO: 1
-TestWF:
-  a:
-    erase: true
-    resources:
-      cpus: 7
-  b:
-    erase: true
-    resources:
-      cpus: 2
-  c:
-    erase: false
-    resources:
-      cpus: 10
-  d:
-    resources:
-      cpus: 15
-    EOF
-
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
-    Log.with_severity 3 do
-      orchestrator.process(rules, jobs)
-    end
-
-    jobs.each do |job|
-      assert job.step(:c).dependencies.empty?
-      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
-      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
-    end
-
-  end
-
-  def test_orchestrate_top_level
-
-    jobs =[]
-
-    num = 3
-    num.times do |i|
-      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
-      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:c, name + " #{i}") }
-    end
-    jobs.each do |j| j.recursive_clean end
-
-    rules = YAML.load <<-EOF
-defaults:
-  erase: true
-  log: 4
-default_resources:
-  IO: 1
-TestWF:
-  a:
-    resources:
-      cpus: 7
-  b:
-    resources:
-      cpus: 2
-  c:
-    resources:
-      cpus: 10
-  d:
-    resources:
-      cpus: 15
-    EOF
-
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
-    Log.with_severity 3 do
-      orchestrator.process(rules, jobs)
-    end
-
-    jobs.each do |job|
-      next unless job.task_name.to_s == 'd'
-      assert job.step(:c).dependencies.empty?
-      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/a/")}.any?
-      assert job.step(:c).info[:archived_info].keys.select{|k| k.include?("TestWF/b/")}.any?
-    end
-
-  end
-
-  def test_orchestrate_top_level_double_dep
-
-    jobs =[]
-
-    num = 10
-    num.times do |i|
-      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:e, name + " #{i}") }
-      jobs.concat %w(TEST1 TEST2).collect{|name| TestWF.job(:d, name + " #{i}") }
-    end
-    jobs.each do |j| j.recursive_clean end
-
-    rules = YAML.load <<-EOF
-defaults:
-  erase: true
-  log: 4
-default_resources:
-  IO: 1
-TestWF:
-  a:
-    resources:
-      cpus: 7
-  b:
-    resources:
-      cpus: 2
-  c:
-    resources:
-      cpus: 10
-  d:
-    resources:
-      cpus: 15
-    EOF
-
-    orchestrator = Workflow::Orchestrator.new(TestWF::MULT, "cpus" => 30, "IO" => 4, "size" => 10 )
-    Log.with_severity 3 do
-      orchestrator.process(rules, jobs)
-    end
-
-    jobs.each do |job|
-      next unless job.task_name.to_s == 'd' || job.task_name.to_s == 'e'
-      assert job.info[:archived_info].keys.select{|k| k.include?("TestWF/c/")}.any?
-      assert job.info[:archived_info].keys.select{|k| k.include?("TestWF/c/")}.any?
-    end
-
-  end
-end
-