rbbt-util 5.32.7 → 5.32.9

data/share/rbbt_commands/slurm/orchestrate

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'rbbt/hpc/orchestrate'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-si--singularity_img* Singularity image to use
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
+-q--queue* Queue
+-t--task_cpus* Tasks
+-W--workflows* Additional workflows
+-tm--time* Time
+-OR--orchestration_rules* Orchestration rules
+-rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+EOF
+
+batch_system = $slurm_options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/slurm/tail

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt slurm tail <directory> [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-r--running Running jobs only
+-q--queued Queued jobs only
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+-p--progress Report progress of job and the dependencies
+-SBP--sbatch_parameters show sbatch parameters
+-PERF--procpath_performance show Procpath performance summary
+-sacct--sacct_peformance show sacct performance summary
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+directory = ARGV.shift
+
+raise ParameterException if directory.nil?
+
+directory = File.dirname(directory) unless File.directory?(directory)
+
+require 'rbbt/hpc/slurm'
+
+HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/slurm/task

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-si--singularity_img* Singularity image to use
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
+-q--queue* Queue
+-t--task_cpus* Tasks
+-W--workflows* Additional workflows
+-tm--time* Time
+-rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+EOF
+
+batch_system = $slurm_options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.7
+  version: 5.32.9
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-04-30 00:00:00.000000000 Z
+date: 2021-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -165,7 +165,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Utilities for handling tsv files, caches, etc
-email: miguel.vazquez@cnio.es
+email: miguel.vazquez.g@bsc.es
 executables:
 - rbbt_query.rb
 - rbbt_exec.rb
@@ -378,6 +378,11 @@ files:
 - share/rbbt_commands/hpc/tail
 - share/rbbt_commands/hpc/task
 - share/rbbt_commands/log
+- share/rbbt_commands/lsf/clean
+- share/rbbt_commands/lsf/list
+- share/rbbt_commands/lsf/orchestrate
+- share/rbbt_commands/lsf/tail
+- share/rbbt_commands/lsf/task
 - share/rbbt_commands/migrate
 - share/rbbt_commands/migrate_job
 - share/rbbt_commands/purge_job
@@ -387,6 +392,11 @@ files:
 - share/rbbt_commands/resource/produce
 - share/rbbt_commands/resource/read
 - share/rbbt_commands/rsync
+- share/rbbt_commands/slurm/clean
+- share/rbbt_commands/slurm/list
+- share/rbbt_commands/slurm/orchestrate
+- share/rbbt_commands/slurm/tail
+- share/rbbt_commands/slurm/task
 - share/rbbt_commands/stat/abs
 - share/rbbt_commands/stat/boxplot
 - share/rbbt_commands/stat/compare_lists