rbbt-util 5.32.26 → 5.33.0

data/test/rbbt/hpc/orchestrate/test_batches.rb

@@ -0,0 +1,113 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/workflow'
+require 'rbbt/hpc/orchestrate/batches'
+
+module TestWFA
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :a2 => :string do self.task_name.to_s end
+
+  dep :a2
+  task :a3 => :string do self.task_name.to_s end
+end
+
+module TestWFB
+  extend Workflow
+
+  dep TestWFA, :a2
+  task :b1 => :string do self.task_name.to_s end
+
+  dep :b1
+  task :b2 => :string do self.task_name.to_s end
+end
+
+module TestWFC
+  extend Workflow
+
+  dep TestWFA, :a1
+  dep_task :c1, TestWFB, :b2
+
+  task :c2 => :string do self.task_name.to_s end
+
+  dep :c1
+  dep :c2
+  task :c3 => :string do self.task_name.to_s end
+
+  dep_task :c4, TestWFC, :c3
+end
+
+module TestWFD
+  extend Workflow
+
+  dep TestWFC, :c3, :jobname => "First c3"
+  dep TestWFC, :c3, :jobname => "Second c3"
+  task :d1 => :string do self.task_name.to_s end
+end
+
+class TestOrchestrate < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a_b:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1, TestWFA#a2
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1
+  chain_d:
+    tasks: TestWFD#d1, TestWFC#c1, TestWFC#c2, TestWFC#c3
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+    time: 10min
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+    time: 10s
+  EOF
+
+
+  def test_job_batches_d
+    job = TestWFD.job(:d1, nil)
+    job.recursive_clean
+
+    batches = HPC::Orchestration.job_batches(RULES, job)
+    assert_equal 3, batches.length
+  end
+
+  def test_job_batches_c3
+    job = TestWFC.job(:c3, nil)
+    job.recursive_clean
+
+    batches = HPC::Orchestration.job_batches(RULES, job)
+  end
+
+  def test_job_batches_c4
+    job = TestWFC.job(:c4, nil)
+    job.recursive_clean
+
+    batches = HPC::Orchestration.job_batches(RULES, job)
+    assert_equal 3, batches.length
+  end
+
+end

data/test/rbbt/hpc/orchestrate/test_chains.rb

@@ -0,0 +1,139 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/workflow'
+require 'rbbt/hpc/orchestrate/chains'
+
+module TestWFA
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :a2 => :string do self.task_name.to_s end
+
+  dep :a2
+  task :a3 => :string do self.task_name.to_s end
+end
+
+module TestWFB
+  extend Workflow
+
+  dep TestWFA, :a2
+  task :b1 => :string do self.task_name.to_s end
+
+  dep :b1
+  task :b2 => :string do self.task_name.to_s end
+end
+
+module TestWFC
+  extend Workflow
+
+  dep TestWFA, :a1
+  dep_task :c1, TestWFB, :b2
+
+  task :c2 => :string do self.task_name.to_s end
+
+  dep :c1
+  dep :c2
+  task :c3 => :string do self.task_name.to_s end
+end
+
+module TestWFD
+  extend Workflow
+
+  dep TestWFC, :c3, :jobname => "First c3"
+  dep TestWFC, :c3, :jobname => "Second c3"
+  task :d1 => :string do self.task_name.to_s end
+end
+
+class TestOrchestrate < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a_b:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1, TestWFA#a2
+    comment: Should not include TestWFA#a1
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    tasks: TestWFB#b1, TestWFB#b2, TestWFA#a1
+  chain_d:
+    tasks: TestWFD#d1, TestWFC#c1, TestWFC#c2, TestWFC#c3
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+  EOF
+
+  def test_parse_chains
+    chains = HPC::Orchestration.parse_chains RULES
+    assert chains[:chain_a][:tasks]["TestWFA"].include?("a1")
+  end
+
+  def test_job_chain_a
+    job = TestWFA.job(:a3, nil)
+    job.recursive_clean
+
+    job_chains = HPC::Orchestration.job_chains(RULES, job)
+    job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
+
+    assert_equal %w(chain_a_b chain_a), job_chains.fields
+    assert_equal 3, job_chains["chain_a"].last[:jobs].length
+    assert_equal job, job_chains["chain_a"].last[:top_level]
+  end
+
+  def test_job_chain_b
+    job = TestWFB.job(:b2, nil)
+    job.recursive_clean
+
+    job_chains = HPC::Orchestration.job_chains(RULES, job)
+    job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
+
+    assert_equal %w(chain_a chain_b chain_b2 chain_a_b).sort, job_chains.fields.sort
+
+    assert_equal 2, job_chains["chain_a"].last[:jobs].length
+    assert_equal job.step("a2"), job_chains["chain_a"].last[:top_level]
+
+    assert_equal 2, job_chains["chain_b"].last[:jobs].length
+    assert_equal job, job_chains["chain_b"].last[:top_level]
+
+    assert_equal 2, job_chains["chain_b2"].last[:jobs].length
+    assert_equal job, job_chains["chain_b2"].last[:top_level]
+
+    assert_equal 4, job_chains["chain_a_b"].last[:jobs].length
+    assert_equal job, job_chains["chain_a_b"].last[:top_level]
+  end
+  
+  def test_job_chains_double
+    job = TestWFD.job(:d1, nil)
+    job.recursive_clean
+
+    job_chains = HPC::Orchestration.job_chains(RULES, job)
+    job_chains = NamedArray.setup(job_chains, job_chains.collect{|n,i| n })
+
+    assert_equal 2, job_chains.select{|n,i| n == 'chain_a' }.length
+    assert_equal ["First c3", "Second c3"].sort, job_chains.
+      select{|n,i| n == 'chain_a' }.
+      collect{|n,i| i[:top_level].name }.sort
+
+    assert_equal 1, job_chains.select{|n,i| n == 'chain_d' }.length
+  end
+end
+

data/test/rbbt/hpc/orchestrate/test_rules.rb

@@ -0,0 +1,92 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/workflow'
+require 'rbbt/hpc/orchestrate/rules'
+
+module TestWFA
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :a2 => :string do self.task_name.to_s end
+
+  dep :a2
+  task :a3 => :string do self.task_name.to_s end
+end
+
+module TestWFB
+  extend Workflow
+
+  dep TestWFA, :a2
+  task :b1 => :string do self.task_name.to_s end
+
+  dep :b1
+  task :b2 => :string do self.task_name.to_s end
+end
+
+module TestWFC
+  extend Workflow
+
+  dep TestWFA, :a1
+  dep_task :c1, TestWFB, :b2
+
+  task :c2 => :string do self.task_name.to_s end
+
+  dep :c1
+  dep :c2
+  task :c3 => :string do self.task_name.to_s end
+end
+
+class TestOrchestrate < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    comment: This chain is not valid, it is missing a2
+    tasks: TestWFA#a1, TestWFA#a3, TestWFB#b1, TestWFB#b2
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+  EOF
+
+  def test_defaults
+    rules = HPC::Orchestration.task_specific_rules RULES, "TestWFA", :a1
+    assert_equal "first_queue" , rules[:queue]
+  end
+
+
+  def test_task_options
+    rules = HPC::Orchestration.task_specific_rules RULES, "TestWFA", :a1
+    assert_equal 10, rules[:cpus]
+    assert_equal 4, rules[:log]
+  end
+
+  def test_skip
+    rules = HPC::Orchestration.task_specific_rules RULES, "TestWFC", :c1
+    assert rules[:skip]
+  end
+
+
+end
+

data/test/rbbt/hpc/test_orchestrate.rb

@@ -0,0 +1,144 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/workflow'
+require 'rbbt/hpc/orchestrate'
+
+module TestWFA
+  extend Workflow
+
+  task :a1 => :string do self.task_name.to_s end
+
+  dep :a1
+  task :a2 => :string do self.task_name.to_s end
+
+  dep :a2
+  task :a3 => :string do self.task_name.to_s end
+end
+
+module TestWFB
+  extend Workflow
+
+  dep TestWFA, :a2
+  task :b1 => :string do self.task_name.to_s end
+
+  dep :b1
+  task :b2 => :string do self.task_name.to_s end
+end
+
+module TestWFC
+  extend Workflow
+
+  dep TestWFA, :a1
+  dep_task :c1, TestWFB, :b2
+
+  task :c2 => :string do self.task_name.to_s end
+
+  dep :c1
+  dep :c2
+  task :c3 => :string do self.task_name.to_s end
+end
+
+class TestOrchestrate < Test::Unit::TestCase
+
+  RULES = IndiferentHash.setup(YAML.load(<<-EOF))
+---
+defaults:
+  queue: first_queue
+  time: 1h
+  log: 2
+  config_keys: key1 value1 token1
+chains:
+  chain_a:
+    workflow: TestWFA
+    tasks: a1, a2, a3
+    config_keys: key2 value2 token2, key3 value3 token3.1 token3.2
+  chain_b:
+    workflow: TestWFB
+    tasks: b1, b2
+  chain_b2:
+    comment: This chain is not valid, it is missing a2
+    tasks: TestWFA#a1, TestWFA#a3, TestWFB#b1, TestWFB#b2
+TestWFA:
+  defaults:
+    log: 4
+    config_keys: key4 value4 token4
+  a1:
+    cpus: 10
+    config_keys: key5 value5 token5
+TestWFC:
+  defaults:
+    skip: true
+    log: 4
+  EOF
+
+  def test_parse_chains
+    chains = HPC::Orchestration.parse_chains RULES
+    assert chains[:chain_a][:tasks]["TestWFA"].include?("a1")
+  end
+
+  def test_task_options
+    rules = HPC::Orchestration.task_specific_rules RULES, "TestWFA", :a1
+    assert_equal 10, rules[:cpus]
+    assert_equal 4, rules[:log]
+  end
+
+  def test_job_workload
+    job = TestWFB.job(:b2, nil)
+    job.recursive_clean
+
+    workload = HPC::Orchestration.job_workload(job)
+
+    assert workload.include? job
+  end
+
+  def test_job_chains
+    job = TestWFB.job(:b2, nil)
+
+    job.recursive_clean
+    job_chains = HPC::Orchestration.job_chains(RULES, job)
+    assert_equal 3, job_chains.length
+
+    TestWFB.job(:b1).run
+    job_chains = HPC::Orchestration.job_chains(RULES, job)
+    assert_equal 0, job_chains.length
+
+    job.recursive_clean
+    TestWFA.job(:a1).run
+    job_chains = HPC::Orchestration.job_chains(RULES, job)
+    assert_equal 2, job_chains.length
+  end
+
+  def test_job_batches
+    job = TestWFB.job(:b2, nil)
+
+    job.recursive_clean
+    batches = HPC::Orchestration.job_batches(RULES, job)
+    assert_equal 2, batches.length
+
+    assert_equal job.rec_dependencies.length + 1, batches.inject(0){|acc,e| acc += e[:jobs].length }
+  end
+
+  def test_job_batches_skip
+    job = TestWFC.job(:c1, nil)
+
+    job.recursive_clean
+    batches = HPC::Orchestration.job_batches(RULES, job)
+    assert_equal 2, batches.length
+
+    assert_equal job.rec_dependencies.length + 1, batches.inject(0){|acc,e| acc += e[:jobs].length }
+  end
+
+  def test_job_batches_skip_2
+    job = TestWFC.job(:c3, nil)
+
+    job.recursive_clean
+    batches = HPC::Orchestration.job_batches(RULES, job)
+    assert_equal 4, batches.length
+
+    assert_equal job.rec_dependencies.length + 1, batches.inject(0){|acc,e| acc += e[:jobs].length }
+
+    batches
+  end
+
+
+end
+

data/test/rbbt/util/test_misc.rb

@@ -604,8 +604,12 @@ EOF
     assert_equal 10, Misc.timespan('10')
     assert_equal 10, Misc.timespan('10s')
     assert_equal 60, Misc.timespan('1min')
+    assert_equal 60, Misc.timespan('00:01:00')
+    assert_equal 60, Misc.timespan('01:00')
+    assert_equal 60, Misc.timespan('0:1:00')
     assert_equal 60*60*24, Misc.timespan('1d')
     assert_equal 60*60*24, Misc.timespan('1d')
+    assert_equal 60*60*24, Misc.timespan('24:00:00')
   end
 
   def test_remove_long_items

data/test/rbbt/workflow/step/test_dependencies.rb

@@ -203,20 +203,21 @@ class TestWorkflowDependency < Test::Unit::TestCase
     size = 100000
     content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
-    Log.severity = 0
-    TmpFile.with_file(content) do |input_file|
-      begin
-        job = DepWorkflow.job(:s3, "TEST", :input_file => input_file)
-        job.recursive_clean
-        job.run(:stream)
-        io = TSV.get_stream job
-        while line = io.gets
-          last_line = line.strip
+    Log.with_severity 0 do
+      TmpFile.with_file(content) do |input_file|
+        begin
+          job = DepWorkflow.job(:s3, "TEST", :input_file => input_file)
+          job.recursive_clean
+          job.run(:stream)
+          io = TSV.get_stream job
+          while line = io.gets
+            last_line = line.strip
+          end
+          io.join if io.respond_to? :join
+        rescue Exception
+          job.abort
+          raise $!
         end
-        io.join if io.respond_to? :join
-      rescue Exception
-        job.abort
-        raise $!
       end
     end
     assert last_line.include? "Line #{size}"

data/test/rbbt/workflow/step/test_save_load_inputs.rb

@@ -0,0 +1,46 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/workflow/step/save_load_inputs'
+
+ENV["RBBT_DEBUG_JOB_HASH"] = true.to_s
+require 'rbbt/workflow'
+module TestSaveLoadWF
+  extend Workflow
+
+  task :number => :integer do
+    10
+  end
+
+  task :list => :array do
+    (0..10).to_a.collect{|e| e.to_s}
+  end
+
+  input :list, :array
+  input :number, :integer
+  task :reverse => :array do |list|
+    list.reverse
+  end
+
+  dep :list
+  dep :number
+  dep :reverse, :list => :list, :number => :number
+  task :prefix => :array do
+    step(:reverse).run.collect{|e| "A-#{e}" }
+  end
+end
+
+class TestSaveLoad < Test::Unit::TestCase
+  def test_save
+    Log.with_severity 0 do
+    job = TestSaveLoadWF.job(:prefix)
+    job.recursive_clean
+    job = TestSaveLoadWF.job(:prefix)
+    TmpFile.with_file do |directory|
+      Step.save_job_inputs(job.step(:reverse), directory)
+      job.produce
+      newjob = TestSaveLoadWF.job_for_directory_inputs(:reverse, directory)
+      assert_equal job.rec_dependencies.last.path, newjob.path
+    end
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.26
+  version: 5.33.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-10 00:00:00.000000000 Z
+date: 2022-02-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -215,7 +215,11 @@ files:
 - lib/rbbt/hpc.rb
 - lib/rbbt/hpc/batch.rb
 - lib/rbbt/hpc/lsf.rb
+- lib/rbbt/hpc/orchestrate.old.rb
 - lib/rbbt/hpc/orchestrate.rb
+- lib/rbbt/hpc/orchestrate/batches.rb
+- lib/rbbt/hpc/orchestrate/chains.rb
+- lib/rbbt/hpc/orchestrate/rules.rb
 - lib/rbbt/hpc/slurm.rb
 - lib/rbbt/knowledge_base.rb
 - lib/rbbt/knowledge_base/enrichment.rb
@@ -345,6 +349,7 @@ files:
 - lib/rbbt/workflow/step/info.rb
 - lib/rbbt/workflow/step/prepare.rb
 - lib/rbbt/workflow/step/run.rb
+- lib/rbbt/workflow/step/save_load_inputs.rb
 - lib/rbbt/workflow/step/status.rb
 - lib/rbbt/workflow/task.rb
 - lib/rbbt/workflow/usage.rb
@@ -470,7 +475,11 @@ files:
 - test/rbbt/association/test_open.rb
 - test/rbbt/association/test_util.rb
 - test/rbbt/entity/test_identifiers.rb
+- test/rbbt/hpc/orchestrate/test_batches.rb
+- test/rbbt/hpc/orchestrate/test_chains.rb
+- test/rbbt/hpc/orchestrate/test_rules.rb
 - test/rbbt/hpc/test_batch.rb
+- test/rbbt/hpc/test_orchestrate.rb
 - test/rbbt/hpc/test_slurm.rb
 - test/rbbt/knowledge_base/test_enrichment.rb
 - test/rbbt/knowledge_base/test_entity.rb
@@ -550,6 +559,7 @@ files:
 - test/rbbt/util/test_simpleopt.rb
 - test/rbbt/util/test_tmpfile.rb
 - test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/workflow/step/test_save_load_inputs.rb
 - test/rbbt/workflow/test_doc.rb
 - test/rbbt/workflow/test_remote_workflow.rb
 - test/rbbt/workflow/test_schedule.rb
@@ -592,6 +602,7 @@ test_files:
 - test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/workflow/step/test_save_load_inputs.rb
 - test/rbbt/workflow/test_task.rb
 - test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb
@@ -649,7 +660,11 @@ test_files:
 - test/rbbt/tsv/test_stream.rb
 - test/rbbt/test_association.rb
 - test/rbbt/hpc/test_batch.rb
+- test/rbbt/hpc/orchestrate/test_chains.rb
+- test/rbbt/hpc/orchestrate/test_rules.rb
+- test/rbbt/hpc/orchestrate/test_batches.rb
 - test/rbbt/hpc/test_slurm.rb
+- test/rbbt/hpc/test_orchestrate.rb
 - test/rbbt/association/test_database.rb
 - test/rbbt/association/test_item.rb
 - test/rbbt/association/test_open.rb