rbbt-util 5.32.7 → 5.32.13

data/share/rbbt_commands/slurm/orchestrate

@@ -0,0 +1,61 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'rbbt/hpc/orchestrate'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-si--singularity_img* Singularity image to use
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
+-q--queue* Queue
+-t--task_cpus* Tasks
+-tm--time* Time
+-m--mem* SLURM minimum memory
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
+-OR--orchestration_rules* Orchestration rules
+-rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+EOF
+
+batch_system = $slurm_options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/slurm/tail

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/hpc'
+
+#$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Queue a job in Marenostrum
+
+$ rbbt slurm tail <directory> [options] 
+
+-h--help Print this help
+-d--done Done jobs only
+-e--error Error jobs only
+-a--aborted SLURM aboted jobs
+-r--running Running jobs only
+-q--queued Queued jobs only
+-j--job* Job ids
+-s--search* Regular expression
+-t--tail* Show the last lines of the STDERR
+-p--progress Report progress of job and the dependencies
+-SBP--sbatch_parameters show sbatch parameters
+-PERF--procpath_performance show Procpath performance summary
+-sacct--sacct_peformance show sacct performance summary
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+EOF
+
+if options[:help]
+ if defined? rbbt_usage
+  rbbt_usage 
+ else
+  puts SOPT.doc
+ end
+ exit 0
+end
+
+batch_system = options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+directory = ARGV.shift
+
+raise ParameterException if directory.nil?
+
+directory = File.dirname(directory) unless File.directory?(directory)
+
+require 'rbbt/hpc/slurm'
+
+HPC::BATCH_MODULE.follow_job directory, true

data/share/rbbt_commands/slurm/task

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-si--singularity_img* Singularity image to use
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-pd--purge_deps Purge job dependencies
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-BPP--batch_procpath* Save Procpath performance for batch job; specify only options
+-q--queue* Queue
+-t--task_cpus* Tasks
+-tm--time* Time
+-m--mem* SLURM minimum memory
+-mcpu--mem_per_cpu* SLURM minimum memory per CPU
+-lin--licenses* SLURM licenses
+-cons--constraint* SLURM constraint
+-W--workflows* Additional workflows
+-rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
+-bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
+EOF
+
+batch_system = $slurm_options.delete :batch_system
+batch_system ||= 'auto'
+
+HPC::BATCH_MODULE = HPC.batch_system batch_system
+
+raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/workflow/forget_deps

@@ -38,11 +38,11 @@ step.copy_files_dir
 dependencies = step.dependencies
 
 if remove && remove.any?
- remove_paths = dependencies.select do |dep|
-  remove.include?(dep.task_name) || remove.include?([dep.workflow.to_s, dep.task_name] * "#")
- end.collect{|dep| dep.path } 
+  remove_paths = dependencies.select do |dep|
+    remove.include?(dep.task_name) || remove.include?([dep.workflow.to_s, dep.task_name] * "#")
+  end.collect{|dep| dep.path } 
 else
- remove_paths = dependencies.collect{|dep| dep.path }
+  remove_paths = dependencies.collect{|dep| dep.path }
 end
 
 step.set_info :dependencies, step.info[:dependencies].reject{|info| remove_paths.include? info.last}
@@ -50,6 +50,7 @@ step.set_info :dependencies, step.info[:dependencies].reject{|info| remove_paths
 if options[:purge] || options[:recursive_purge]
   dependencies.each do |dependency|
     next unless remove_paths.include? dependency.path
+    next unless Open.exists?(dependency.info_file)
     Step.purge(dependency.path, options[:recursive_purge])
   end
 end

data/test/rbbt/util/test_python.rb

@@ -2,6 +2,7 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helpe
 require 'rbbt/util/python'
 
 class TestPython < Test::Unit::TestCase
+
   def test_python
     TmpFile.with_file do |tmpdir|
       code =<<-EOF
@@ -67,7 +68,7 @@ def python_print():
 
   def test_keras
     defined = RbbtPython.run do
-      pyimport "keras.models", as: :km
+      pyimport "tensorflow.keras.models", as: :km
       defined?(km.Sequential)
     end
     assert defined
@@ -75,7 +76,7 @@ def python_print():
 
   def test_keras_import
     defined = RbbtPython.run do
-      pyfrom "keras.models", import: :Sequential
+      pyfrom "tensorflow.keras.models", import: :Sequential
       defined?(self::Sequential)
     end
     assert defined

data/test/rbbt/util/test_simpleDSL.rb

@@ -35,12 +35,12 @@ class TestDSL < Test::Unit::TestCase
 
   def test_config
     config = <<-EOC
-  action1("Hello")
-  action2("Good bye")
+  action1 "Hello"  
+  action2 "Good bye" 
     EOC
 
     begin
-      assert(@parser.config(:action) == config)
+      assert_equal config.split("\n").collect{|l| l.strip}, @parser.config(:action).split("\n").collect{|l| l.strip}
     rescue SimpleDSL::NoRuby2Ruby
     end
   end

data/test/rbbt/workflow/util/test_data.rb

@@ -0,0 +1,35 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/workflow/util/data'
+
+module TestDataWF
+  extend Workflow
+  extend Workflow::Data
+
+  input :name, :string
+  input :salutation, :string
+  task :salute => :string do |name,salutation|
+    "Hi #{name}: #{salutation}"
+  end
+
+  data_task :salute_data, TestDataWF, :salute do |directory,options|
+    options.merge({:salutation => directory.salutation.read})
+  end
+end
+
+class TestWorkflowData < Test::Unit::TestCase
+  def test_workflow_data
+    TmpFile.with_file do |tmpdir|
+      tmpdir = Path.setup(tmpdir.dup)
+
+      Open.write(tmpdir.TestDir.options.name, "Miguel")
+      Open.write(tmpdir.TestDir.salutation, "My salutations")
+
+      TestDataWF.data tmpdir
+
+      job = TestDataWF.job(:salute_data, "TestDir")
+      job.recursive_clean.run
+      ppp job.run
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.32.7
+  version: 5.32.13
 platform: ruby
 authors:
 - Miguel Vazquez
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-30 00:00:00.000000000 Z
+date: 2021-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -165,7 +165,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Utilities for handling tsv files, caches, etc
-email: miguel.vazquez@cnio.es
+email: miguel.vazquez.g@bsc.es
 executables:
 - rbbt_query.rb
 - rbbt_exec.rb
@@ -344,9 +344,11 @@ files:
 - lib/rbbt/workflow/task.rb
 - lib/rbbt/workflow/usage.rb
 - lib/rbbt/workflow/util/archive.rb
+- lib/rbbt/workflow/util/data.rb
 - lib/rbbt/workflow/util/orchestrator.rb
 - lib/rbbt/workflow/util/provenance.rb
 - lib/rbbt/workflow/util/trace.rb
+- python/rbbt.py
 - share/Rlib/plot.R
 - share/Rlib/svg.R
 - share/Rlib/util.R
@@ -378,6 +380,11 @@ files:
 - share/rbbt_commands/hpc/tail
 - share/rbbt_commands/hpc/task
 - share/rbbt_commands/log
+- share/rbbt_commands/lsf/clean
+- share/rbbt_commands/lsf/list
+- share/rbbt_commands/lsf/orchestrate
+- share/rbbt_commands/lsf/tail
+- share/rbbt_commands/lsf/task
 - share/rbbt_commands/migrate
 - share/rbbt_commands/migrate_job
 - share/rbbt_commands/purge_job
@@ -387,6 +394,11 @@ files:
 - share/rbbt_commands/resource/produce
 - share/rbbt_commands/resource/read
 - share/rbbt_commands/rsync
+- share/rbbt_commands/slurm/clean
+- share/rbbt_commands/slurm/list
+- share/rbbt_commands/slurm/orchestrate
+- share/rbbt_commands/slurm/tail
+- share/rbbt_commands/slurm/task
 - share/rbbt_commands/stat/abs
 - share/rbbt_commands/stat/boxplot
 - share/rbbt_commands/stat/compare_lists
@@ -539,13 +551,14 @@ files:
 - test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/test_task.rb
 - test/rbbt/workflow/util/test_archive.rb
+- test/rbbt/workflow/util/test_data.rb
 - test/rbbt/workflow/util/test_orchestrator.rb
 - test/test_helper.rb
 homepage: http://github.com/mikisvaz/rbbt-util
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -560,104 +573,104 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.6
-signing_key:
+rubygems_version: 3.1.4
+signing_key: 
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/test_helper.rb
-- test/rbbt/entity/test_identifiers.rb
-- test/rbbt/test_resource.rb
-- test/rbbt/test_association.rb
-- test/rbbt/hpc/test_slurm.rb
-- test/rbbt/hpc/test_batch.rb
-- test/rbbt/persist/test_tsv.rb
-- test/rbbt/persist/tsv/test_kyotocabinet.rb
-- test/rbbt/persist/tsv/test_lmdb.rb
-- test/rbbt/persist/tsv/test_tokyocabinet.rb
-- test/rbbt/persist/tsv/test_cdb.rb
-- test/rbbt/persist/tsv/test_leveldb.rb
-- test/rbbt/persist/tsv/test_sharder.rb
-- test/rbbt/test_packed_index.rb
 - test/rbbt/test_entity.rb
-- test/rbbt/test_fix_width_table.rb
 - test/rbbt/workflow/test_remote_workflow.rb
-- test/rbbt/workflow/test_doc.rb
-- test/rbbt/workflow/step/test_dependencies.rb
-- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/util/test_archive.rb
 - test/rbbt/workflow/util/test_orchestrator.rb
-- test/rbbt/workflow/test_task.rb
+- test/rbbt/workflow/util/test_data.rb
+- test/rbbt/workflow/test_doc.rb
+- test/rbbt/workflow/test_schedule.rb
 - test/rbbt/workflow/test_step.rb
-- test/rbbt/test_tsv.rb
-- test/rbbt/test_annotations.rb
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/util/test_migrate.rb
-- test/rbbt/util/test_simpleDSL.rb
-- test/rbbt/util/concurrency/processes/test_socket.rb
-- test/rbbt/util/concurrency/test_processes.rb
-- test/rbbt/util/concurrency/test_threads.rb
-- test/rbbt/util/test_filecache.rb
-- test/rbbt/util/simpleopt/test_get.rb
-- test/rbbt/util/simpleopt/test_parse.rb
-- test/rbbt/util/simpleopt/test_setup.rb
-- test/rbbt/util/test_misc.rb
-- test/rbbt/util/test_excel2tsv.rb
-- test/rbbt/util/test_semaphore.rb
-- test/rbbt/util/test_procpath.rb
-- test/rbbt/util/R/test_model.rb
-- test/rbbt/util/R/test_eval.rb
-- test/rbbt/util/R/test_plot.rb
-- test/rbbt/util/test_open.rb
-- test/rbbt/util/test_tmpfile.rb
-- test/rbbt/util/test_cmd.rb
-- test/rbbt/util/test_concurrency.rb
+- test/rbbt/workflow/step/test_dependencies.rb
+- test/rbbt/workflow/test_task.rb
+- test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb
-- test/rbbt/util/test_config.rb
-- test/rbbt/util/test_log.rb
-- test/rbbt/util/test_simpleopt.rb
-- test/rbbt/util/test_python.rb
-- test/rbbt/util/test_chain_methods.rb
+- test/rbbt/util/test_procpath.rb
+- test/rbbt/util/misc/test_development.rb
 - test/rbbt/util/misc/test_omics.rb
+- test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_format.rb
 - test/rbbt/util/misc/test_lock.rb
 - test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/misc/test_bgzf.rb
-- test/rbbt/util/misc/test_development.rb
-- test/rbbt/util/misc/test_format.rb
-- test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/test_concurrency.rb
+- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/R/test_plot.rb
+- test/rbbt/util/R/test_eval.rb
+- test/rbbt/util/R/test_model.rb
+- test/rbbt/util/test_config.rb
+- test/rbbt/util/test_log.rb
+- test/rbbt/util/test_simpleDSL.rb
 - test/rbbt/util/log/test_progress.rb
+- test/rbbt/util/test_tmpfile.rb
 - test/rbbt/util/test_R.rb
-- test/rbbt/test_workflow.rb
-- test/rbbt/knowledge_base/test_entity.rb
-- test/rbbt/knowledge_base/test_enrichment.rb
-- test/rbbt/knowledge_base/test_traverse.rb
-- test/rbbt/knowledge_base/test_syndicate.rb
-- test/rbbt/knowledge_base/test_registry.rb
-- test/rbbt/knowledge_base/test_query.rb
-- test/rbbt/association/test_item.rb
-- test/rbbt/association/test_util.rb
-- test/rbbt/association/test_open.rb
-- test/rbbt/association/test_database.rb
-- test/rbbt/association/test_index.rb
-- test/rbbt/annotations/test_util.rb
-- test/rbbt/test_monitor.rb
-- test/rbbt/resource/test_path.rb
-- test/rbbt/test_hpc.rb
-- test/rbbt/test_persist.rb
+- test/rbbt/util/test_excel2tsv.rb
+- test/rbbt/util/test_misc.rb
+- test/rbbt/util/test_open.rb
+- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/simpleopt/test_parse.rb
+- test/rbbt/util/simpleopt/test_setup.rb
+- test/rbbt/util/simpleopt/test_get.rb
+- test/rbbt/util/test_python.rb
+- test/rbbt/util/test_filecache.rb
+- test/rbbt/util/concurrency/test_processes.rb
+- test/rbbt/util/concurrency/test_threads.rb
+- test/rbbt/util/concurrency/processes/test_socket.rb
+- test/rbbt/util/test_semaphore.rb
+- test/rbbt/util/test_chain_methods.rb
+- test/rbbt/util/test_migrate.rb
+- test/rbbt/test_resource.rb
+- test/rbbt/test_packed_index.rb
+- test/rbbt/tsv/test_change_id.rb
+- test/rbbt/tsv/test_attach.rb
+- test/rbbt/tsv/test_filter.rb
 - test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_csv.rb
-- test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/tsv/test_accessor.rb
+- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_field_index.rb
 - test/rbbt/tsv/test_util.rb
-- test/rbbt/tsv/test_accessor.rb
-- test/rbbt/tsv/test_filter.rb
-- test/rbbt/tsv/test_stream.rb
+- test/rbbt/tsv/test_index.rb
 - test/rbbt/tsv/test_parallel.rb
-- test/rbbt/tsv/test_matrix.rb
-- test/rbbt/tsv/test_attach.rb
+- test/rbbt/tsv/test_manipulate.rb
 - test/rbbt/tsv/test_excel.rb
-- test/rbbt/tsv/test_change_id.rb
-- test/rbbt/tsv/test_index.rb
-- test/rbbt/tsv/parallel/test_traverse.rb
 - test/rbbt/tsv/parallel/test_through.rb
+- test/rbbt/tsv/parallel/test_traverse.rb
+- test/rbbt/tsv/test_stream.rb
+- test/rbbt/test_association.rb
+- test/rbbt/hpc/test_batch.rb
+- test/rbbt/hpc/test_slurm.rb
+- test/rbbt/association/test_database.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/association/test_open.rb
+- test/rbbt/association/test_util.rb
+- test/rbbt/association/test_index.rb
+- test/rbbt/test_knowledge_base.rb
+- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_cdb.rb
+- test/rbbt/persist/tsv/test_lmdb.rb
+- test/rbbt/persist/tsv/test_sharder.rb
+- test/rbbt/persist/tsv/test_leveldb.rb
+- test/rbbt/persist/tsv/test_tokyocabinet.rb
+- test/rbbt/persist/test_tsv.rb
+- test/rbbt/test_tsv.rb
+- test/rbbt/test_annotations.rb
+- test/rbbt/test_fix_width_table.rb
+- test/rbbt/test_workflow.rb
+- test/rbbt/entity/test_identifiers.rb
+- test/rbbt/annotations/test_util.rb
+- test/rbbt/test_hpc.rb
+- test/rbbt/test_monitor.rb
+- test/rbbt/test_persist.rb
+- test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_registry.rb
+- test/rbbt/knowledge_base/test_syndicate.rb
+- test/rbbt/knowledge_base/test_query.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
+- test/rbbt/knowledge_base/test_traverse.rb
+- test/test_helper.rb