rbbt-util 5.30.10 → 5.31.1

data/share/rbbt_commands/hpc/orchestrate

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'rbbt/hpc/orchestrate'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
+-q--queue* Queue
+-t--task_cpus* Tasks
+-W--workflows* Additional workflows
+-tm--time* Time
+-OR--orchestration_rules* Orchestration rules
+-rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+EOF
+
+batch_system = $slurm_options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = case batch_system.to_s.downcase
+               when 'slurm'
+                 HPC::SLURM
+               when 'lsf'
+                 HPC::LSF
+               when 'auto'
+                 case $previous_commands.last
+                 when 'slurm'
+                   HPC::SLURM
+                 when 'lsf'
+                   HPC::LSF
+                 else
+                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
+                   when 'slurm'
+                     HPC::SLURM
+                   when 'lsf'
+                     HPC::LSF
+                   else
+                     case ENV["BATCH_SYSTEM"].to_s.downcase
+                     when 'slurm'
+                       HPC::SLURM
+                     when 'lsf'
+                       HPC::LSF
+                     end
+                   end
+                 end
+               end
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/hpc/tail

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt slurm tail <directory> [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-r--running Running jobs only
+-q--queued Queued jobs only
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+-p--progress Report progress of job and the dependencies
+-SBP--sbatch_parameters show sbatch parameters
+-PERF--procpath_performance show Procpath performance summary
+-sacct--sacct_peformance show sacct performance summary
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = case batch_system.to_s.downcase
+               when 'slurm'
+                 HPC::SLURM
+               when 'lsf'
+                 HPC::LSF
+               when 'auto'
+                 case $previous_commands.last
+                 when 'slurm'
+                   HPC::SLURM
+                 when 'lsf'
+                   HPC::LSF
+                 else
+                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
+                   when 'slurm'
+                     HPC::SLURM
+                   when 'lsf'
+                     HPC::LSF
+                   else
+                     case ENV["BATCH_SYSTEM"].to_s.downcase
+                     when 'slurm'
+                       HPC::SLURM
+                     when 'lsf'
+                       HPC::LSF
+                     end
+                   end
+                 end
+               end
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+directory = ARGV.shift
+
+raise ParameterException if directory.nil?
+
+directory = File.dirname(directory) unless File.directory?(directory)
+
+require 'rbbt/hpc/slurm'
+
+HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/hpc/task

@@ -0,0 +1,80 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
+-q--queue* Queue
+-t--task_cpus* Tasks
+-W--workflows* Additional workflows
+-tm--time* Time
+-rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+EOF
+
+batch_system = $slurm_options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = case batch_system.to_s.downcase
+               when 'slurm'
+                 HPC::SLURM
+               when 'lsf'
+                 HPC::LSF
+               when 'auto'
+                 case $previous_commands.last
+                 when 'slurm'
+                   HPC::SLURM
+                 when 'lsf'
+                   HPC::LSF
+                 else
+                   case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
+                   when 'slurm'
+                     HPC::SLURM
+                   when 'lsf'
+                     HPC::LSF
+                   else
+                     case ENV["BATCH_SYSTEM"].to_s.downcase
+                     when 'slurm'
+                       HPC::SLURM
+                     when 'lsf'
+                       HPC::LSF
+                     end
+                   end
+                 end
+               end
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find

data/test/rbbt/hpc/test_batch.rb

@@ -0,0 +1,65 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/hpc/batch'
+require 'rbbt/workflow'
+
+Workflow.require_workflow "Sample"
+Workflow.require_workflow "HTS"
+class TestSLURM < Test::Unit::TestCase
+
+  def _test_batch_options
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    TmpFile.with_file do |batch_dir|
+
+      options = HPC::BATCH.batch_options(job, :batch_dir => batch_dir, :batch_modules => 'java')
+
+      iii options
+    end
+  end
+
+  def _test_template
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    TmpFile.with_file do |batch_dir|
+
+      template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java')
+      ppp template
+
+    end
+  end
+
+  def _test_template_singularity
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    TmpFile.with_file do |batch_dir|
+
+      template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :singularity => true)
+      ppp template
+
+    end
+  end
+
+  def _test_template_contain
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    TmpFile.with_file do |batch_dir|
+
+      template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :contain_and_sync => true, :wipe_container => 'force')
+      ppp template
+
+    end
+  end
+  
+  def test_template_singularity_contain
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    TmpFile.with_file do |batch_dir|
+
+      template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :contain_and_sync => true, :wipe_container => 'force', :singularity => true)
+      ppp template
+
+    end
+  end
+  
+end
+

data/test/rbbt/hpc/test_slurm.rb

@@ -0,0 +1,30 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/hpc/slurm'
+require 'rbbt/workflow'
+
+Workflow.require_workflow "Sample"
+Workflow.require_workflow "HTS"
+class TestSLURM < Test::Unit::TestCase
+
+  def _test_template
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    TmpFile.with_file do |batch_dir|
+
+      template = HPC::SLURM.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java')
+      ppp template
+
+    end
+  end
+
+  def test_run_job
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    job.clean
+
+    jobid = HPC::SLURM.run_job(job, :workflows => "HTS", :batch_modules => 'java', :env_cmd => '_JAVA_OPTIONS="-Xms1g -Xmx${MAX_MEMORY}m"', :queue => :debug, :time => '01:00:00', :config_keys => "HTS_light", :task_cpus => '10', :tail => true, :clean_task => "HTS#mutect2")
+    iii jobid
+  end
+
+end
+

data/test/rbbt/util/misc/test_development.rb

@@ -0,0 +1,11 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/misc/development'
+
+class TestMiscDevelopment < Test::Unit::TestCase
+  def test_ssh
+    Misc.ssh_run 'mn1' do
+      puts "hola"
+    end
+  end
+end
+

data/test/test_helper.rb

@@ -20,7 +20,9 @@ class Test::Unit::TestCase
   def setup
     Random.new
 
-    Persist.cachedir = Rbbt.tmp.test.persistence.find :user if defined? Persist
+    if defined? Persist
+      Persist.cachedir = Rbbt.tmp.test.persistence.find(:user)
+    end
 
     Entity.entity_property_cache = Rbbt.tmp.test.entity_property.find(:user) if defined? Entity
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.30.10
+  version: 5.31.1
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-16 00:00:00.000000000 Z
+date: 2021-03-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -211,6 +211,8 @@ files:
 - lib/rbbt/entity/identifiers.rb
 - lib/rbbt/fix_width_table.rb
 - lib/rbbt/hpc.rb
+- lib/rbbt/hpc/batch.rb
+- lib/rbbt/hpc/lsf.rb
 - lib/rbbt/hpc/orchestrate.rb
 - lib/rbbt/hpc/slurm.rb
 - lib/rbbt/knowledge_base.rb
@@ -369,6 +371,11 @@ files:
 - share/rbbt_commands/file_server/add
 - share/rbbt_commands/file_server/list
 - share/rbbt_commands/file_server/remove
+- share/rbbt_commands/hpc/clean
+- share/rbbt_commands/hpc/list
+- share/rbbt_commands/hpc/orchestrate
+- share/rbbt_commands/hpc/tail
+- share/rbbt_commands/hpc/task
 - share/rbbt_commands/log
 - share/rbbt_commands/migrate
 - share/rbbt_commands/migrate_job
@@ -379,10 +386,6 @@ files:
 - share/rbbt_commands/resource/produce
 - share/rbbt_commands/resource/read
 - share/rbbt_commands/rsync
-- share/rbbt_commands/slurm/clean
-- share/rbbt_commands/slurm/list
-- share/rbbt_commands/slurm/orchestrate
-- share/rbbt_commands/slurm/task
 - share/rbbt_commands/stat/abs
 - share/rbbt_commands/stat/boxplot
 - share/rbbt_commands/stat/compare_lists
@@ -449,6 +452,8 @@ files:
 - test/rbbt/association/test_open.rb
 - test/rbbt/association/test_util.rb
 - test/rbbt/entity/test_identifiers.rb
+- test/rbbt/hpc/test_batch.rb
+- test/rbbt/hpc/test_slurm.rb
 - test/rbbt/knowledge_base/test_enrichment.rb
 - test/rbbt/knowledge_base/test_entity.rb
 - test/rbbt/knowledge_base/test_query.rb
@@ -499,6 +504,7 @@ files:
 - test/rbbt/util/concurrency/test_threads.rb
 - test/rbbt/util/log/test_progress.rb
 - test/rbbt/util/misc/test_bgzf.rb
+- test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_format.rb
 - test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_multipart_payload.rb
@@ -551,7 +557,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
@@ -567,6 +573,7 @@ test_files:
 - test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb
 - test/rbbt/util/test_procpath.rb
+- test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/misc/test_format.rb
@@ -617,6 +624,8 @@ test_files:
 - test/rbbt/tsv/parallel/test_traverse.rb
 - test/rbbt/tsv/test_stream.rb
 - test/rbbt/test_association.rb
+- test/rbbt/hpc/test_batch.rb
+- test/rbbt/hpc/test_slurm.rb
 - test/rbbt/association/test_database.rb
 - test/rbbt/association/test_item.rb
 - test/rbbt/association/test_open.rb