rbbt-util 5.43.0 → 5.44.0

data/test/rbbt/workflow/step/test_dependencies.rb

@@ -1,295 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/workflow'
-
-module DepWorkflow
-  extend Workflow
-
-  input :input_file, :file, "Input file", nil, :stream => true
-  task :s1 => :array do |input_file|
-    TSV.traverse input_file, :type => :array, :into => :stream, :bar => "Task1" do |line|
-      line + "\t" << "Task1"
-    end
-  end
-
-  dep :s1
-  task :s2 => :array do |input_file|
-    TSV.traverse step(:s1), :type => :array, :into => :stream, :bar => "Task2" do |line|
-      next [line.split("\t").first, Misc::SKIP_TAG] * "\t" if rand < 0.9
-      line + "\t" << "Task2"
-    end
-  end
-
-  dep :s1
-  dep :s2
-  task :s3 => :array do |input_file|
-    Misc.paste_streams(dependencies.reverse)
-  end
-
-  input :input_file, :file, "Input file", nil, :stream => true
-  task :task1 => :array do |input_file|
-    TSV.traverse input_file, :type => :array, :into => :stream, :bar => "Task1" do |line|
-      line + "\t" << "Task1"
-    end
-  end
-
-  dep :task1
-  task :task2 => :array do
-    TSV.traverse step(:task1), :type => :array, :into => :stream, :bar => "Task2" do |line|
-      line + "\t" << "Task2"
-    end
-  end
-
-  dep :task1
-  task :task3 => :array do
-    TSV.traverse step(:task1), :type => :array, :into => :stream, :bar => "Task3" do |line|
-      line + "\t" << "Task3"
-    end
-  end
-
-  dep :task2
-  dep :task3
-  task :task4 => :array do
-    Misc.paste_streams(dependencies)
-  end
-
-  dep :task4
-  task :task5 => :array do
-    TSV.traverse step(:task4), :type => :array, :into => :stream, :bar => "Task5" do |line|
-      line + "\t" << "Task5"
-    end
-  end
-
-  dep :task2
-  dep :task5
-  task :task6 => :array do
-    Misc.paste_streams(dependencies)
-  end
-
-  input :stream_file, :file, "Streamed file", nil, :stream => true
-  task :task7 => :array do |file|
-    TSV.traverse file, :type => :array, :into => :stream, :bar => "Task7" do |line|
-      line + "\t" << "Task7"
-    end
-  end
-
-  dep :task6
-  dep :task7, :stream_file => :task6
-  task :task8 => :array do
-    TSV.get_stream step(:task7)
-  end
-end
-
-module ComputeWorkflow
-  extend Workflow
-
-  input :input_file, :file, "Input file", nil, :stream => true
-  task :task1 => :array do |input_file|
-    TSV.traverse input_file, :type => :array, :into => :stream, :bar => "Task1" do |line|
-      line + "\t" << "Task1"
-    end
-  end
-
-  dep :task1, :compute => :produce
-  task :task2 => :array do
-    TSV.traverse step(:task1), :type => :array, :into => :stream, :bar => "Task2" do |line|
-      line + "\t" << "Task2"
-    end
-  end
-
-end
-
-module ResumeWorkflow
-  extend Workflow
-
-  resumable
-  task :resume => :string do
-    if file('foo').exists?
-      'done'
-    else
-      Open.mkdir files_dir
-      Open.touch(file('foo'))
-      raise 
-    end
-  end
-
-  dep :resume
-  task :reverse => :string do
-    step(:resume).load.reverse
-  end
-end
-
-class TestWorkflowDependency < Test::Unit::TestCase
-  def test_task1
-    size = 10000
-    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
-    TmpFile.with_file(content) do |input_file|
-      job = DepWorkflow.job(:task1, "TEST", :input_file => input_file)
-      io = TSV.get_stream job.run(:stream)
-      last_line = nil
-      while line = io.gets
-        last_line = line.strip
-      end
-      io.join
-
-      assert_equal "Line #{size}\tTask1", last_line
-    end
-  end
-
-  def test_task2
-    size = 10000
-    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
-    TmpFile.with_file(content) do |input_file|
-      job = DepWorkflow.job(:task2, "TEST", :input_file => input_file)
-      io = TSV.get_stream job.run(:stream)
-      last_line = nil
-      while line = io.gets
-        last_line = line.strip
-      end
-      io.join
-
-      assert_equal "Line #{size}\tTask1\tTask2", last_line
-    end
-  end
-
-  def test_task3
-    size = 100000
-    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
-    TmpFile.with_file(content) do |input_file|
-      job = DepWorkflow.job(:task3, "TEST", :input_file => input_file)
-      io = TSV.get_stream job.run(:stream)
-      last_line = nil
-      while line = io.gets
-        last_line = line.strip
-      end
-      io.join
-
-      assert_equal "Line #{size}\tTask1\tTask3", last_line
-    end
-  end
-
-  def test_task4
-    size = 100000
-    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
-    last_line = nil
-    TmpFile.with_file(content) do |input_file|
-      job = DepWorkflow.job(:task4, "TEST", :input_file => input_file)
-      io = TSV.get_stream job.run(:stream) 
-      while line = io.gets 
-        last_line = line.strip
-      end
-      io.join
-    end
-
-    assert_equal "Line #{size}\tTask1\tTask2\tTask1\tTask3", last_line
-  end
-  
-  def test_task5
-    size = 10000
-    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
-    last_line = nil
-    TmpFile.with_file(content) do |input_file|
-      job = DepWorkflow.job(:task5, "TEST", :input_file => input_file)
-      io = TSV.get_stream job.run(:stream)
-      while line = io.gets
-        last_line = line.strip
-      end
-      io.join
-    end
-    assert_equal "Line #{size}\tTask1\tTask2\tTask1\tTask3\tTask5", last_line
-  end
-  
-  def test_s3
-    size = 100000
-    content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
-    last_line = nil
-    Log.with_severity 0 do
-      TmpFile.with_file(content) do |input_file|
-        begin
-          job = DepWorkflow.job(:s3, "TEST", :input_file => input_file)
-          job.recursive_clean
-          job.run(:stream)
-          io = TSV.get_stream job
-          while line = io.gets
-            last_line = line.strip
-          end
-          io.join if io.respond_to? :join
-        rescue Exception
-          job.abort
-          raise $!
-        end
-      end
-    end
-    assert last_line.include? "Line #{size}"
-  end
-  
-  def test_task6
-    size = 100000
-    content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
-    last_line = nil
-    TmpFile.with_file(content) do |input_file|
-      begin
-        job = DepWorkflow.job(:task6, "TEST", :input_file => input_file)
-        job.recursive_clean
-        job.run(:stream)
-        io = TSV.get_stream job
-        while line = io.gets
-          last_line = line.strip
-        end
-        io.join
-      rescue Exception
-        job.abort
-        raise $!
-      end
-    end
-    assert_equal "Line #{size}\tTask1\tTask2\tTask1\tTask2\tTask1\tTask3\tTask5", last_line
-  end
-  
-  def test_task8
-    size = 10000
-    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
-    last_line = nil
-    TmpFile.with_file(content) do |input_file|
-      begin
-      job = DepWorkflow.job(:task8, "TEST", :input_file => input_file)
-      job.run(:stream)
-      io = TSV.get_stream job
-      while line = io.gets
-        last_line = line.strip
-      end
-      io.join
-      rescue Exception
-        job.abort
-        raise $!
-      end
-    end
-    assert_equal "Line #{size}\tTask1\tTask2\tTask1\tTask2\tTask1\tTask3\tTask5\tTask7", last_line
-  end
-
-  def test_compute
-    size = 10000
-    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
-    TmpFile.with_file(content) do |input_file|
-      job = ComputeWorkflow.job(:task2, "TEST", :input_file => input_file)
-      io = TSV.get_stream job.run(:stream)
-      last_line = nil
-      while line = io.gets
-        last_line = line.strip
-      end
-      io.join
-
-      assert_equal "Line #{size}\tTask1\tTask2", last_line
-    end
-  end
-
-  def test_resume
-    job = ResumeWorkflow.job(:reverse)
-    job.recursive_clean
-    assert_raise do
-      job.run
-    end
-    assert job.dependencies.first.file('foo').exists?
-    assert_equal 'done'.reverse, job.run
-  end
-
-end
-

data/test/rbbt/workflow/step/test_save_load_inputs.rb

@@ -1,136 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
-require 'rbbt/workflow/step/save_load_inputs'
-
-ENV["RBBT_DEBUG_JOB_HASH"] = true.to_s
-require 'rbbt/workflow'
-module TestSaveLoadWF
-  extend Workflow
-
-  task :number => :integer do
-    10
-  end
-
-  task :list => :array do
-    (0..10).to_a.collect{|e| e.to_s}
-  end
-
-  input :list, :array
-  input :number, :integer
-  task :reverse => :array do |list|
-    list.reverse
-  end
-
-  dep :list
-  dep :number
-  dep :reverse, :list => :list, :number => :number
-  task :prefix => :array do
-    step(:reverse).run.collect{|e| "A-#{e}" }
-  end
-
-  input :integer, :integer
-  input :float, :float
-  input :file, :file
-  input :file_no_file, :file, "", nil, :nofile => true
-  input :string, :string
-  input :select, :select
-  input :array, :array
-  input :array_no_file, :array, "", nil, :nofile => true
-  input :tsv, :tsv, "", nil, :nofile => true
-  input :tsv_no_file, :tsv, "", nil, :nofile => true
-  input :binary, :binary, "", nil, :nofile => true
-  input :binary_no_file, :tsv, "", nil, :nofile => true
-  task :save_test => :string do
-    "DONE"
-  end
-end
-
-class TestSaveLoad < Test::Unit::TestCase
-  def test_save
-    job = TestSaveLoadWF.job(:prefix)
-    job.recursive_clean
-    job = TestSaveLoadWF.job(:prefix)
-    TmpFile.with_file do |directory|
-      Step.save_job_inputs(job.step(:reverse), directory)
-      job.produce
-      newjob = TestSaveLoadWF.job_for_directory_inputs(:reverse, directory)
-      assert_equal job.rec_dependencies.last.path, newjob.path
-    end
-  end
-
-  def test_save_all_normal
-    tsv = TSV.setup({}, "Key~Value#:type=:single")
-    tsv["key"] = "value"
-    options = {
-      :float => 0.5,
-      :integer => 15,
-      :string => "STRING",
-      :select => "option",
-      :array => [0,1,2,3],
-      :tsv => tsv,
-    }
-    Log.with_severity 0 do
-      Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
-        job = TestSaveLoadWF.job(:save_test, nil, options)
-        TmpFile.with_file do |directory|
-          Step.save_job_inputs(job, directory)
-          newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
-          assert_equal job.path, newjob.path
-        end
-      end
-    end
-  end
-
-  def test_save_all_file
-    tsv = TSV.setup({}, "Key~Value#:type=:single")
-    tsv["key"] = "value"
-    Log.with_severity 0 do
-      Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
-        TmpFile.with_file do |input_file|
-          Path.setup(input_file)
-          options = {
-            :float => input_file.float,
-            :integer => input_file.integer,
-            :string => input_file.string,
-            :select => input_file.select,
-            :array => input_file.array,
-            :tsv => input_file.tsv_file,
-          }
-          job = TestSaveLoadWF.job(:save_test, nil, options)
-          TmpFile.with_file do |directory|
-            Step.save_job_inputs(job, directory)
-            newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
-            assert_equal job.path, newjob.path
-          end
-        end
-      end
-    end
-  end
-
-  def test_save_all_job_file
-    tsv = TSV.setup({}, "Key~Value#:type=:single")
-    tsv["key"] = "value"
-    Log.with_severity 0 do
-      Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
-        TmpFile.with_file do |input_file|
-          Path.setup(input_file)
-          options = {
-            :float => input_file.float,
-            :integer => input_file.integer,
-            :string => input_file.string,
-            :select => input_file.select,
-            :array => TestSaveLoadWF.job(:list),
-            :binary => TestSaveLoadWF.job(:list).file('binary'),
-            :tsv => input_file.tsv_file,
-          }
-          job = TestSaveLoadWF.job(:save_test, nil, options)
-          TmpFile.with_file do |directory|
-            Step.save_job_inputs(job, directory)
-            newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
-            assert_equal job.path, newjob.path
-          end
-        end
-      end
-    end
-  end
-end
-

data/test/rbbt/workflow/test_doc.rb

@@ -1,30 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
-require 'rbbt/workflow'
-
-class TestWorkflowDoc < Test::Unit::TestCase
-  TEMPLATE=<<-EOF
-Title of the template
-
-Paragraph1
-
-Paragraph2
-
-# Tasks
-
-Paragraph3
-
-## task1
-
-Task 1
-
-## task2
-
-Task 2
-
-  EOF
-  def test_parse
-    i = Workflow.parse_workflow_doc(TEMPLATE)
-    assert_equal "Title of the template", i[:title]
-  end
-end
-

data/test/rbbt/workflow/test_remote_workflow.rb

@@ -1,99 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
-require 'rbbt/workflow'
-require 'rbbt/workflow/remote_workflow'
-
-class TestWFRest
-  extend Workflow
-
-  input :name, :string, "Name to greet", "World"
-  task :hi => :string do |name|
-    "Hello #{name}"
-  end
-
-  dep :hi
-  task :intro => :string do 
-    step(:hi).load + ", nice to meet you"
-  end
-end
-
-class TestRemoteWorkflow < Test::Unit::TestCase
-
-  def remote_workflow_server(workflow, options = {}, &block)
-    trap(:USR1){ raise TestServerLoaded}
-
-    begin
-      pid = Process.fork do 
-        TmpFile.with_file do |app_dir|
-          Misc.in_dir(app_dir) do
-            require 'rack'
-            ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
-
-            app_dir = Path.setup(app_dir.dup)
-            Open.write(app_dir.etc.target_workflow.find, workflow.to_s)
-
-            config_ru_file = File.exist?('./workflow_config.ru') ? './workflow_config.ru' : Rbbt.share['workflow_config.ru'].find
-            options[:config] = config_ru_file
-            app = Rack::Server.new(options)
-            app.start do
-              Process.kill :USR1, Process.ppid
-            end
-          end
-        end
-      end
-
-      begin
-        sleep 1 while true
-      rescue TestServerLoaded
-      end
-
-      client = RemoteWorkflow.new "http://localhost:#{options[:Port] || 9292}/#{workflow.to_s}", workflow.to_s
-
-      yield client
-
-    rescue
-      Log.exception $!
-    ensure
-      Process.kill :INT, pid
-      Process.wait pid
-    end
-  end
-
-  def test_rest
-    keyword_test :rest do
-
-      remote_workflow_server(TestWFRest) do |client|
-        job = client.job(:hi, nil, {})
-        job.clean
-        job = client.job(:hi, nil, {})
-        assert ! job.done?
-        job.run
-        job.produce
-        job = client.job(:hi, nil, {})
-        assert job.done?
-        sleep 1
-      end
-
-      remote_workflow_server(TestWFRest) do |client|
-        assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
-        assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
-        assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
-      end
-
-      remote_workflow_server(TestWFRest, :Port => 1902) do |client|
-        assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
-        assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
-        assert_equal "Hello Miguel, nice to meet you", client.job(:intro, nil, {:name => :Miguel}).run.chomp
-      end
-    end
-  end
-
-
-  def test_ssh
-    keyword_test :ssh do
-      client = RemoteWorkflow.new "ssh://#{ENV["HOSTNAME"] || 'localhost'}:Translation", "Translation"
-      job = client.job("translate", "SSH-TEST-1", :genes => ["TP53","KRAS"])
-      assert_equal 2, job.run.select{|l| l =~ /ENSG/}.length
-    end
-  end
-end
-