rbbt-util 5.30.9 → 5.31.0

data/test/rbbt/hpc/test_batch.rb

@@ -0,0 +1,65 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/hpc/batch'
+require 'rbbt/workflow'
+
+Workflow.require_workflow "Sample"
+Workflow.require_workflow "HTS"
+class TestSLURM < Test::Unit::TestCase
+
+  def _test_batch_options
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    TmpFile.with_file do |batch_dir|
+
+      options = HPC::BATCH.batch_options(job, :batch_dir => batch_dir, :batch_modules => 'java')
+
+      iii options
+    end
+  end
+
+  def _test_template
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    TmpFile.with_file do |batch_dir|
+
+      template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java')
+      ppp template
+
+    end
+  end
+
+  def _test_template_singularity
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    TmpFile.with_file do |batch_dir|
+
+      template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :singularity => true)
+      ppp template
+
+    end
+  end
+
+  def _test_template_contain
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    TmpFile.with_file do |batch_dir|
+
+      template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :contain_and_sync => true, :wipe_container => 'force')
+      ppp template
+
+    end
+  end
+  
+  def test_template_singularity_contain
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    TmpFile.with_file do |batch_dir|
+
+      template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :contain_and_sync => true, :wipe_container => 'force', :singularity => true)
+      ppp template
+
+    end
+  end
+  
+end
+

data/test/rbbt/hpc/test_slurm.rb

@@ -0,0 +1,30 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/hpc/slurm'
+require 'rbbt/workflow'
+
+Workflow.require_workflow "Sample"
+Workflow.require_workflow "HTS"
+class TestSLURM < Test::Unit::TestCase
+
+  def _test_template
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    TmpFile.with_file do |batch_dir|
+
+      template = HPC::SLURM.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java')
+      ppp template
+
+    end
+  end
+
+  def test_run_job
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    job.clean
+
+    jobid = HPC::SLURM.run_job(job, :workflows => "HTS", :batch_modules => 'java', :env_cmd => '_JAVA_OPTIONS="-Xms1g -Xmx${MAX_MEMORY}m"', :queue => :debug, :time => '01:00:00', :config_keys => "HTS_light", :task_cpus => '10', :tail => true, :clean_task => "HTS#mutect2")
+    iii jobid
+  end
+
+end
+

data/test/rbbt/util/misc/test_development.rb

@@ -0,0 +1,11 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/misc/development'
+
+class TestMiscDevelopment < Test::Unit::TestCase
+  def test_ssh
+    Misc.ssh_run 'mn1' do
+      puts "hola"
+    end
+  end
+end
+

data/test/rbbt/util/test_config.rb

@@ -5,8 +5,9 @@ class TestConfig < Test::Unit::TestCase
   def setup
     Rbbt::Config.set({:cpus => 30}, :test_config, :test)
     Rbbt::Config.set(:cpus, 5, "slow::2", :test)
-    Rbbt::Config.set({:token => "token"}, "token", "key:token")
+    Rbbt::Config.set({:token => "token"}, "token")
     Rbbt::Config.set(:notoken, "no_token")
+    Rbbt::Config.set({:emptytoken => "empty"})
   end
 
   def test_simple
@@ -19,8 +20,12 @@ class TestConfig < Test::Unit::TestCase
 
   def test_simple_no_token
     assert_equal "token", Rbbt::Config.get("token", "token")
-    assert_equal "token", Rbbt::Config.get("token")
-    assert_equal "no_token", Rbbt::Config.get("notoken")
+    assert_equal "no_token", Rbbt::Config.get("notoken", "key:notoken")
+    assert_equal 'token', Rbbt::Config.get("token", "key:token")
+    assert_equal 'token', Rbbt::Config.get("token")
+    assert_equal nil, Rbbt::Config.get("token", "someotherthing")
+    assert_equal "default_token", Rbbt::Config.get("token", 'unknown', :default => 'default_token')
+    assert_equal 'empty', Rbbt::Config.get("emptytoken", 'key:emptytoken')
   end
 
   def test_prio
@@ -54,6 +59,11 @@ class TestConfig < Test::Unit::TestCase
     assert_equal "V1", Rbbt::Config.get('key', 'token1', 'token2')
   end
 
+  def test_default
+    Rbbt::Config.add_entry 'key', 'V1', 'token1'
+    assert_equal "V3", Rbbt::Config.get('key', 'token2', :default => 'V3')
+  end
+
 
 end
 

data/test/test_helper.rb

@@ -20,7 +20,9 @@ class Test::Unit::TestCase
   def setup
     Random.new
 
-    Persist.cachedir = Rbbt.tmp.test.persistence.find :user if defined? Persist
+    if defined? Persist
+      Persist.cachedir = Rbbt.tmp.test.persistence.find(:user)
+    end
 
     Entity.entity_property_cache = Rbbt.tmp.test.entity_property.find(:user) if defined? Entity
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.30.9
+  version: 5.31.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-11 00:00:00.000000000 Z
+date: 2021-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -211,6 +211,8 @@ files:
 - lib/rbbt/entity/identifiers.rb
 - lib/rbbt/fix_width_table.rb
 - lib/rbbt/hpc.rb
+- lib/rbbt/hpc/batch.rb
+- lib/rbbt/hpc/lsf.rb
 - lib/rbbt/hpc/orchestrate.rb
 - lib/rbbt/hpc/slurm.rb
 - lib/rbbt/knowledge_base.rb
@@ -369,6 +371,11 @@ files:
 - share/rbbt_commands/file_server/add
 - share/rbbt_commands/file_server/list
 - share/rbbt_commands/file_server/remove
+- share/rbbt_commands/hpc/clean
+- share/rbbt_commands/hpc/list
+- share/rbbt_commands/hpc/orchestrate
+- share/rbbt_commands/hpc/tail
+- share/rbbt_commands/hpc/task
 - share/rbbt_commands/log
 - share/rbbt_commands/migrate
 - share/rbbt_commands/migrate_job
@@ -379,10 +386,6 @@ files:
 - share/rbbt_commands/resource/produce
 - share/rbbt_commands/resource/read
 - share/rbbt_commands/rsync
-- share/rbbt_commands/slurm/clean
-- share/rbbt_commands/slurm/list
-- share/rbbt_commands/slurm/orchestrate
-- share/rbbt_commands/slurm/task
 - share/rbbt_commands/stat/abs
 - share/rbbt_commands/stat/boxplot
 - share/rbbt_commands/stat/compare_lists
@@ -449,6 +452,8 @@ files:
 - test/rbbt/association/test_open.rb
 - test/rbbt/association/test_util.rb
 - test/rbbt/entity/test_identifiers.rb
+- test/rbbt/hpc/test_batch.rb
+- test/rbbt/hpc/test_slurm.rb
 - test/rbbt/knowledge_base/test_enrichment.rb
 - test/rbbt/knowledge_base/test_entity.rb
 - test/rbbt/knowledge_base/test_query.rb
@@ -499,6 +504,7 @@ files:
 - test/rbbt/util/concurrency/test_threads.rb
 - test/rbbt/util/log/test_progress.rb
 - test/rbbt/util/misc/test_bgzf.rb
+- test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_format.rb
 - test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_multipart_payload.rb
@@ -551,7 +557,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
@@ -567,6 +573,7 @@ test_files:
 - test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb
 - test/rbbt/util/test_procpath.rb
+- test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/misc/test_format.rb
@@ -617,6 +624,8 @@ test_files:
 - test/rbbt/tsv/parallel/test_traverse.rb
 - test/rbbt/tsv/test_stream.rb
 - test/rbbt/test_association.rb
+- test/rbbt/hpc/test_batch.rb
+- test/rbbt/hpc/test_slurm.rb
 - test/rbbt/association/test_database.rb
 - test/rbbt/association/test_item.rb
 - test/rbbt/association/test_open.rb

data/share/rbbt_commands/slurm/orchestrate

@@ -1,48 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'rbbt/util/simpleopt'
-require 'rbbt/workflow'
-require 'rbbt/workflow/usage'
-require 'rbbt/hpc'
-require 'rbbt/hpc/orchestrate'
-require 'time'
-
-$slurm_options = SOPT.get <<EOF
--dr--dry_run Print only the template
--cj--clean_job Clean job
---drbbt* Use development version of rbbt 
--sing--singularity Use Singularity
--ug--user_group* Use alternative user group for group project directory
--c--contain* Contain in directory (using Singularity)
--s--sync* Contain in directory and sync jobs
--e--exclusive Make exclusive use of the node
--hm--highmem Make use of highmem cores
--wc--wipe_container* Wipe the jobs from the contain directory
--CS--contain_and_sync Contain and sync to default locations
--ci--copy_image When using a container directory, copy image there
--t--tail Tail the logs
--SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
--q--queue* Queue
--t--task_cpus* Tasks
--W--workflows* Additional workflows
--tm--time* Time
--OR--orchestration_rules* Orchestration rules
--rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
-EOF
-
-class Step
-    def run(*args)
-        if done?
-            self.load
-        else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::SLURM.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
-        end
-    end
-end
-
-ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
-load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/slurm/task

@@ -1,46 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'rbbt/util/simpleopt'
-require 'rbbt/workflow'
-require 'rbbt/workflow/usage'
-require 'rbbt/hpc'
-require 'time'
-
-$slurm_options = SOPT.get <<EOF
--dr--dry_run Print only the template
--cj--clean_job Clean job
---drbbt* Use development version of rbbt 
--sing--singularity Use Singularity
--ug--user_group* Use alternative user group for group project directory
--c--contain* Contain in directory (using Singularity)
--s--sync* Contain in directory and sync jobs
--e--exclusive Make exclusive use of the node
--hm--highmem Make use of highmem cores
--wc--wipe_container* Wipe the jobs from the contain directory
--CS--contain_and_sync Contain and sync to default locations
--ci--copy_image When using a container directory, copy image there
--t--tail Tail the logs
--SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
--q--queue* Queue
--t--task_cpus* Tasks
--W--workflows* Additional workflows
--tm--time* Time
--rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
-EOF
-
-class Step
-    def run(*args)
-        if done?
-            self.load
-        else
-            begin
-                Log.debug "Issuing SLURM job for #{self.path}"
-                HPC::SLURM.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
-            rescue HPC::SBATCH
-            end
-        end
-    end
-end
-
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
-load Rbbt.share.rbbt_commands.workflow.task.find