rbbt-util 5.39.0 → 5.40.3

data/share/rbbt_commands/pbs/tail

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt slurm tail <directory|jobid> [options] 
+
+-h--help Print this help
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+directory = ARGV.shift
+
+raise ParameterException if directory.nil?
+
+if directory =~ /^[0-9]*$/
+  workdir = File.expand_path('~/rbbt-batch')
+  Path.setup(workdir)
+
+  workdir.glob("**/job.id").each do |file|
+    next unless directory == Open.read(file).strip
+    directory = File.dirname(file)
+    break
+  end
+end
+
+raise ParameterException, "Could not identify job #{directory}" unless File.exist?(directory)
+
+require 'rbbt/hpc/slurm'
+
+command_txt = Open.read(File.join(directory, 'command.batch'))
+if m = command_txt.match(/#STEP_PATH: (.*)/)
+  step_path = m[1]
+else
+  step_path = nil
+end
+
+puts Log.color(:magenta, "Directory: ") + directory if directory
+puts Log.color(:magenta, "Step path: ") + step_path if step_path
+
+HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/pbs/task

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/workflow/remote_workflow'
+require 'rbbt/hpc'
+require 'time'
+
+rbbt_options = SOPT::GOT_OPTIONS
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-si--singularity_img* Singularity image to use
+-sm--singularity_mounts* Singularity image to use
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
+-q--queue* Queue
+-a--account* Account
+-p--partition* Partition
+-t--task_cpus* Tasks
+-tm--time* Time
+-m--mem* minimum memory
+--gres* Generic resources
+-mcpu--mem_per_cpu* minimum memory per CPU
+-lin--licenses* licenses
+-cons--constraint* constraint
+-W--workflows* Additional workflows
+-rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+-lmod--lua_modules* Lua Modules to load 
+-co--conda* Conda environment to use
+-OR--orchestration_rules* Orchestration rules
+EOF
+
+batch_system = $slurm_options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+class Step
+
+  def run(*args)
+    if done?
+      self.load
+    else
+      begin
+        Log.debug "Issuing BATCH job for #{self.path}"
+        HPC::BATCH_MODULE.run_job(self, $slurm_options) 
+      rescue HPC::BATCH_DRY_RUN
+      end
+    end
+  end
+end
+
+module RemoteStep::SSH
+
+  def _run
+    RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, $slurm_options)
+  end
+
+end
+
+SOPT.current_options = rbbt_options
+load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/slurm/list

@@ -253,6 +253,8 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
       text = CMD.cmd('grep "^#SBATCH" ', :in => Open.read(fcmd)).read.strip
     when 'lsf'
       text = CMD.cmd('grep "^#BSUB" ', :in => Open.read(fcmd)).read.strip
+    when 'pbs'
+      text = CMD.cmd('grep "^#PBS" ', :in => Open.read(fcmd)).read.strip
     else
       text = ""
     end

data/share/rbbt_commands/slurm/orchestrate

@@ -73,7 +73,7 @@ class Step
           join
           self.load
         end
-      rescue HPC::SBATCH
+      rescue HPC::BATCH_DRY_RUN
       end
     end
   end

data/share/rbbt_commands/slurm/task

@@ -32,16 +32,17 @@ $slurm_options = SOPT.get <<EOF
 -p--partition* Partition
 -t--task_cpus* Tasks
 -tm--time* Time
--m--mem* SLURM minimum memory
---gres* SLURM Generic resources
--mcpu--mem_per_cpu* SLURM minimum memory per CPU
--lin--licenses* SLURM licenses
--cons--constraint* SLURM constraint
+-m--mem* minimum memory
+--gres* Generic resources
+-mcpu--mem_per_cpu* minimum memory per CPU
+-lin--licenses* licenses
+-cons--constraint* constraint
 -W--workflows* Additional workflows
 -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
 -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
 -lmod--lua_modules* Lua Modules to load 
 -co--conda* Conda environment to use
+-OR--orchestration_rules* Orchestration rules
 EOF
 
 batch_system = $slurm_options.delete :batch_system
@@ -58,9 +59,9 @@ class Step
       self.load
     else
       begin
-        Log.debug "Issuing SLURM job for #{self.path}"
+        Log.debug "Issuing BATCH job for #{self.path}"
         HPC::BATCH_MODULE.run_job(self, $slurm_options) 
-      rescue HPC::SBATCH
+      rescue HPC::BATCH_DRY_RUN
       end
     end
   end

data/test/rbbt/hpc/test_pbs.rb

@@ -0,0 +1,43 @@
+require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
+require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
+
+require 'rbbt/workflow'
+
+class TestPBS < Test::Unit::TestCase
+  def workflow
+    @workflow ||= Module.new do
+      extend Workflow
+
+      def self.to_s
+        "TestWorkflow"
+      end
+
+      input :name, :string
+      task :hello => :string do |name|
+        "hello #{name}"
+      end
+    end
+  end
+
+  def test_template
+    job = workflow.job(:hello, "TEST", :name => "world")
+
+    TmpFile.with_file do |batch_dir|
+
+      template = HPC::PBS.job_template(job, :batch_dir => batch_dir, :lua_modules => 'java')
+      ppp template
+
+    end
+  end
+
+  def __test_run_job
+    job = Sample.job(:mutect2, "small", :reference => "hg38")
+
+    job.clean
+
+    jobid = HPC::SLURM.run_job(job, :workflows => "HTS", :batch_modules => 'java', :env_cmd => '_JAVA_OPTIONS="-Xms1g -Xmx${MAX_MEMORY}m"', :queue => :debug, :time => '01:00:00', :config_keys => "HTS_light", :task_cpus => '10', :tail => true, :clean_task => "HTS#mutect2")
+    assert jobid.to_s =~ /^\d+$/
+  end
+
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.39.0
+  version: 5.40.3
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-08-02 00:00:00.000000000 Z
+date: 2023-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -136,20 +136,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: nokogiri
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Utilities for handling tsv files, caches, etc
 email: miguel.vazquez.g@bsc.es
 executables:
@@ -206,6 +192,7 @@ files:
 - lib/rbbt/hpc/orchestrate/batches.rb
 - lib/rbbt/hpc/orchestrate/chains.rb
 - lib/rbbt/hpc/orchestrate/rules.rb
+- lib/rbbt/hpc/pbs.rb
 - lib/rbbt/hpc/slurm.rb
 - lib/rbbt/knowledge_base.rb
 - lib/rbbt/knowledge_base/enrichment.rb
@@ -301,6 +288,7 @@ files:
 - lib/rbbt/util/open.rb
 - lib/rbbt/util/procpath.rb
 - lib/rbbt/util/python.rb
+- lib/rbbt/util/python/step.rb
 - lib/rbbt/util/python/util.rb
 - lib/rbbt/util/semaphore.rb
 - lib/rbbt/util/simpleDSL.rb
@@ -389,6 +377,11 @@ files:
 - share/rbbt_commands/lsf/task
 - share/rbbt_commands/migrate
 - share/rbbt_commands/migrate_job
+- share/rbbt_commands/pbs/clean
+- share/rbbt_commands/pbs/list
+- share/rbbt_commands/pbs/orchestrate
+- share/rbbt_commands/pbs/tail
+- share/rbbt_commands/pbs/task
 - share/rbbt_commands/purge_job
 - share/rbbt_commands/resource/claims
 - share/rbbt_commands/resource/exists
@@ -476,6 +469,7 @@ files:
 - test/rbbt/hpc/orchestrate/test_rules.rb
 - test/rbbt/hpc/test_batch.rb
 - test/rbbt/hpc/test_orchestrate.rb
+- test/rbbt/hpc/test_pbs.rb
 - test/rbbt/hpc/test_slurm.rb
 - test/rbbt/knowledge_base/test_enrichment.rb
 - test/rbbt/knowledge_base/test_entity.rb
@@ -590,7 +584,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.13
+rubygems_version: 3.4.19
 signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
@@ -607,6 +601,7 @@ test_files:
 - test/rbbt/hpc/orchestrate/test_rules.rb
 - test/rbbt/hpc/test_batch.rb
 - test/rbbt/hpc/test_orchestrate.rb
+- test/rbbt/hpc/test_pbs.rb
 - test/rbbt/hpc/test_slurm.rb
 - test/rbbt/knowledge_base/test_enrichment.rb
 - test/rbbt/knowledge_base/test_entity.rb