rbbt-util 5.30.13 → 5.31.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. checksums.yaml +4 -4
  2. data/lib/rbbt/hpc.rb +3 -0
  3. data/lib/rbbt/hpc/batch.rb +623 -0
  4. data/lib/rbbt/hpc/lsf.rb +119 -0
  5. data/lib/rbbt/hpc/orchestrate.rb +12 -11
  6. data/lib/rbbt/hpc/slurm.rb +62 -567
  7. data/lib/rbbt/resource/path.rb +3 -1
  8. data/lib/rbbt/tsv/accessor.rb +5 -2
  9. data/lib/rbbt/tsv/dumper.rb +1 -0
  10. data/lib/rbbt/tsv/parallel/traverse.rb +1 -1
  11. data/lib/rbbt/tsv/stream.rb +5 -6
  12. data/lib/rbbt/util/log.rb +22 -1
  13. data/lib/rbbt/util/misc/development.rb +2 -2
  14. data/lib/rbbt/util/misc/options.rb +5 -0
  15. data/lib/rbbt/workflow/step/accessor.rb +1 -1
  16. data/lib/rbbt/workflow/usage.rb +13 -13
  17. data/share/config.ru +3 -3
  18. data/share/rbbt_commands/{slurm → hpc}/clean +91 -18
  19. data/share/rbbt_commands/{slurm → hpc}/list +100 -30
  20. data/share/rbbt_commands/hpc/orchestrate +81 -0
  21. data/share/rbbt_commands/hpc/tail +81 -0
  22. data/share/rbbt_commands/hpc/task +80 -0
  23. data/test/rbbt/hpc/test_batch.rb +65 -0
  24. data/test/rbbt/hpc/test_slurm.rb +30 -0
  25. data/test/rbbt/util/misc/test_development.rb +11 -0
  26. data/test/test_helper.rb +3 -1
  27. metadata +16 -7
  28. data/share/rbbt_commands/slurm/orchestrate +0 -48
  29. data/share/rbbt_commands/slurm/task +0 -46
data/share/rbbt_commands/hpc/orchestrate ADDED
@@ -0,0 +1,81 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rbbt/util/simpleopt'
4
+ require 'rbbt/workflow'
5
+ require 'rbbt/workflow/usage'
6
+ require 'rbbt/hpc'
7
+ require 'rbbt/hpc/orchestrate'
8
+ require 'time'
9
+
10
+ $slurm_options = SOPT.get <<EOF
11
+ -dr--dry_run Print only the template
12
+ -cj--clean_job Clean job
13
+ --drbbt* Use development version of rbbt
14
+ -sing--singularity Use Singularity
15
+ -ug--user_group* Use alternative user group for group project directory
16
+ -c--contain* Contain in directory (using Singularity)
17
+ -s--sync* Contain in directory and sync jobs
18
+ -e--exclusive Make exclusive use of the node
19
+ -hm--highmem Make use of highmem cores
20
+ -wc--wipe_container* Wipe the jobs from the contain directory
21
+ -CS--contain_and_sync Contain and sync to default locations
22
+ -ci--copy_image When using a container directory, copy image there
23
+ -t--tail Tail the logs
24
+ -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
25
+ -q--queue* Queue
26
+ -t--task_cpus* Tasks
27
+ -W--workflows* Additional workflows
28
+ -tm--time* Time
29
+ -OR--orchestration_rules* Orchestration rules
30
+ -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
31
+ EOF
32
+
33
+ batch_system = $slurm_options.delete :batch_system
34
+ batch_system ||= 'auto'
35
+
36
+ HPC::BATCH_MODULE = case batch_system.to_s.downcase
37
+ when 'slurm'
38
+ HPC::SLURM
39
+ when 'lsf'
40
+ HPC::LSF
41
+ when 'auto'
42
+ case $previous_commands.last
43
+ when 'slurm'
44
+ HPC::SLURM
45
+ when 'lsf'
46
+ HPC::LSF
47
+ else
48
+ case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
49
+ when 'slurm'
50
+ HPC::SLURM
51
+ when 'lsf'
52
+ HPC::LSF
53
+ else
54
+ case ENV["BATCH_SYSTEM"].to_s.downcase
55
+ when 'slurm'
56
+ HPC::SLURM
57
+ when 'lsf'
58
+ HPC::LSF
59
+ end
60
+ end
61
+ end
62
+ end
63
+
64
+ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
65
+
66
+ class Step
67
+ def run(*args)
68
+ if done?
69
+ self.load
70
+ else
71
+ begin
72
+ Log.debug "Issuing SLURM job for #{self.path}"
73
+ HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options))
74
+ rescue HPC::SBATCH
75
+ end
76
+ end
77
+ end
78
+ end
79
+
80
+ ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
81
+ load Rbbt.share.rbbt_commands.workflow.task.find
data/share/rbbt_commands/hpc/tail ADDED
@@ -0,0 +1,81 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rbbt-util'
4
+ require 'rbbt/util/simpleopt'
5
+ require 'rbbt/hpc'
6
+
7
+ #$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
8
+
9
+ options = SOPT.setup <<EOF
10
+
11
+ Queue a job in Marenostrum
12
+
13
+ $ rbbt slurm tail <directory> [options]
14
+
15
+ -h--help Print this help
16
+ -d--done Done jobs only
17
+ -e--error Error jobs only
18
+ -a--aborted SLURM aboted jobs
19
+ -r--running Running jobs only
20
+ -q--queued Queued jobs only
21
+ -j--job* Job ids
22
+ -s--search* Regular expression
23
+ -t--tail* Show the last lines of the STDERR
24
+ -p--progress Report progress of job and the dependencies
25
+ -SBP--sbatch_parameters show sbatch parameters
26
+ -PERF--procpath_performance show Procpath performance summary
27
+ -sacct--sacct_peformance show sacct performance summary
28
+ -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
29
+ EOF
30
+
31
+ if options[:help]
32
+ if defined? rbbt_usage
33
+ rbbt_usage
34
+ else
35
+ puts SOPT.doc
36
+ end
37
+ exit 0
38
+ end
39
+
40
+ batch_system = options.delete :batch_system
41
+ batch_system ||= 'auto'
42
+
43
+ HPC::BATCH_MODULE = case batch_system.to_s.downcase
44
+ when 'slurm'
45
+ HPC::SLURM
46
+ when 'lsf'
47
+ HPC::LSF
48
+ when 'auto'
49
+ case $previous_commands.last
50
+ when 'slurm'
51
+ HPC::SLURM
52
+ when 'lsf'
53
+ HPC::LSF
54
+ else
55
+ case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
56
+ when 'slurm'
57
+ HPC::SLURM
58
+ when 'lsf'
59
+ HPC::LSF
60
+ else
61
+ case ENV["BATCH_SYSTEM"].to_s.downcase
62
+ when 'slurm'
63
+ HPC::SLURM
64
+ when 'lsf'
65
+ HPC::LSF
66
+ end
67
+ end
68
+ end
69
+ end
70
+
71
+ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
72
+
73
+ directory = ARGV.shift
74
+
75
+ raise ParameterException if directory.nil?
76
+
77
+ directory = File.dirname(directory) unless File.directory?(directory)
78
+
79
+ require 'rbbt/hpc/slurm'
80
+
81
+ HPC::BATCH_MODULE.follow_job directory, true
data/share/rbbt_commands/hpc/task ADDED
@@ -0,0 +1,80 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rbbt/util/simpleopt'
4
+ require 'rbbt/workflow'
5
+ require 'rbbt/workflow/usage'
6
+ require 'rbbt/hpc'
7
+ require 'time'
8
+
9
+ $slurm_options = SOPT.get <<EOF
10
+ -dr--dry_run Print only the template
11
+ -cj--clean_job Clean job
12
+ --drbbt* Use development version of rbbt
13
+ -sing--singularity Use Singularity
14
+ -ug--user_group* Use alternative user group for group project directory
15
+ -c--contain* Contain in directory (using Singularity)
16
+ -s--sync* Contain in directory and sync jobs
17
+ -e--exclusive Make exclusive use of the node
18
+ -hm--highmem Make use of highmem cores
19
+ -wc--wipe_container* Wipe the jobs from the contain directory
20
+ -CS--contain_and_sync Contain and sync to default locations
21
+ -ci--copy_image When using a container directory, copy image there
22
+ -t--tail Tail the logs
23
+ -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
24
+ -q--queue* Queue
25
+ -t--task_cpus* Tasks
26
+ -W--workflows* Additional workflows
27
+ -tm--time* Time
28
+ -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
29
+ -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
30
+ EOF
31
+
32
+ batch_system = $slurm_options.delete :batch_system
33
+ batch_system ||= 'auto'
34
+
35
+ HPC::BATCH_MODULE = case batch_system.to_s.downcase
36
+ when 'slurm'
37
+ HPC::SLURM
38
+ when 'lsf'
39
+ HPC::LSF
40
+ when 'auto'
41
+ case $previous_commands.last
42
+ when 'slurm'
43
+ HPC::SLURM
44
+ when 'lsf'
45
+ HPC::LSF
46
+ else
47
+ case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
48
+ when 'slurm'
49
+ HPC::SLURM
50
+ when 'lsf'
51
+ HPC::LSF
52
+ else
53
+ case ENV["BATCH_SYSTEM"].to_s.downcase
54
+ when 'slurm'
55
+ HPC::SLURM
56
+ when 'lsf'
57
+ HPC::LSF
58
+ end
59
+ end
60
+ end
61
+ end
62
+
63
+ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
64
+
65
+ class Step
66
+ def run(*args)
67
+ if done?
68
+ self.load
69
+ else
70
+ begin
71
+ Log.debug "Issuing SLURM job for #{self.path}"
72
+ HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options))
73
+ rescue HPC::SBATCH
74
+ end
75
+ end
76
+ end
77
+ end
78
+
79
+ ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
80
+ load Rbbt.share.rbbt_commands.workflow.task.find
data/test/rbbt/hpc/test_batch.rb ADDED
@@ -0,0 +1,65 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
+ require 'rbbt/hpc/batch'
3
+ require 'rbbt/workflow'
4
+
5
+ Workflow.require_workflow "Sample"
6
+ Workflow.require_workflow "HTS"
7
+ class TestSLURM < Test::Unit::TestCase
8
+
9
+ def _test_batch_options
10
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
11
+
12
+ TmpFile.with_file do |batch_dir|
13
+
14
+ options = HPC::BATCH.batch_options(job, :batch_dir => batch_dir, :batch_modules => 'java')
15
+
16
+ iii options
17
+ end
18
+ end
19
+
20
+ def _test_template
21
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
22
+
23
+ TmpFile.with_file do |batch_dir|
24
+
25
+ template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java')
26
+ ppp template
27
+
28
+ end
29
+ end
30
+
31
+ def _test_template_singularity
32
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
33
+
34
+ TmpFile.with_file do |batch_dir|
35
+
36
+ template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :singularity => true)
37
+ ppp template
38
+
39
+ end
40
+ end
41
+
42
+ def _test_template_contain
43
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
44
+
45
+ TmpFile.with_file do |batch_dir|
46
+
47
+ template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :contain_and_sync => true, :wipe_container => 'force')
48
+ ppp template
49
+
50
+ end
51
+ end
52
+
53
+ def test_template_singularity_contain
54
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
55
+
56
+ TmpFile.with_file do |batch_dir|
57
+
58
+ template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :contain_and_sync => true, :wipe_container => 'force', :singularity => true)
59
+ ppp template
60
+
61
+ end
62
+ end
63
+
64
+ end
65
+
data/test/rbbt/hpc/test_slurm.rb ADDED
@@ -0,0 +1,30 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
+ require 'rbbt/hpc/slurm'
3
+ require 'rbbt/workflow'
4
+
5
+ Workflow.require_workflow "Sample"
6
+ Workflow.require_workflow "HTS"
7
+ class TestSLURM < Test::Unit::TestCase
8
+
9
+ def _test_template
10
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
11
+
12
+ TmpFile.with_file do |batch_dir|
13
+
14
+ template = HPC::SLURM.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java')
15
+ ppp template
16
+
17
+ end
18
+ end
19
+
20
+ def test_run_job
21
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
22
+
23
+ job.clean
24
+
25
+ jobid = HPC::SLURM.run_job(job, :workflows => "HTS", :batch_modules => 'java', :env_cmd => '_JAVA_OPTIONS="-Xms1g -Xmx${MAX_MEMORY}m"', :queue => :debug, :time => '01:00:00', :config_keys => "HTS_light", :task_cpus => '10', :tail => true, :clean_task => "HTS#mutect2")
26
+ iii jobid
27
+ end
28
+
29
+ end
30
+
data/test/rbbt/util/misc/test_development.rb ADDED
@@ -0,0 +1,11 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
+ require 'rbbt/util/misc/development'
3
+
4
+ class TestMiscDevelopment < Test::Unit::TestCase
5
+ def test_ssh
6
+ Misc.ssh_run 'mn1' do
7
+ puts "hola"
8
+ end
9
+ end
10
+ end
11
+
data/test/test_helper.rb CHANGED
@@ -20,7 +20,9 @@ class Test::Unit::TestCase
20
20
  def setup
21
21
  Random.new
22
22
 
23
- Persist.cachedir = Rbbt.tmp.test.persistence.find :user if defined? Persist
23
+ if defined? Persist
24
+ Persist.cachedir = Rbbt.tmp.test.persistence.find(:user)
25
+ end
24
26
 
25
27
  Entity.entity_property_cache = Rbbt.tmp.test.entity_property.find(:user) if defined? Entity
26
28
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.30.13
4
+ version: 5.31.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-02-21 00:00:00.000000000 Z
11
+ date: 2021-03-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -211,6 +211,8 @@ files:
211
211
  - lib/rbbt/entity/identifiers.rb
212
212
  - lib/rbbt/fix_width_table.rb
213
213
  - lib/rbbt/hpc.rb
214
+ - lib/rbbt/hpc/batch.rb
215
+ - lib/rbbt/hpc/lsf.rb
214
216
  - lib/rbbt/hpc/orchestrate.rb
215
217
  - lib/rbbt/hpc/slurm.rb
216
218
  - lib/rbbt/knowledge_base.rb
@@ -369,6 +371,11 @@ files:
369
371
  - share/rbbt_commands/file_server/add
370
372
  - share/rbbt_commands/file_server/list
371
373
  - share/rbbt_commands/file_server/remove
374
+ - share/rbbt_commands/hpc/clean
375
+ - share/rbbt_commands/hpc/list
376
+ - share/rbbt_commands/hpc/orchestrate
377
+ - share/rbbt_commands/hpc/tail
378
+ - share/rbbt_commands/hpc/task
372
379
  - share/rbbt_commands/log
373
380
  - share/rbbt_commands/migrate
374
381
  - share/rbbt_commands/migrate_job
@@ -379,10 +386,6 @@ files:
379
386
  - share/rbbt_commands/resource/produce
380
387
  - share/rbbt_commands/resource/read
381
388
  - share/rbbt_commands/rsync
382
- - share/rbbt_commands/slurm/clean
383
- - share/rbbt_commands/slurm/list
384
- - share/rbbt_commands/slurm/orchestrate
385
- - share/rbbt_commands/slurm/task
386
389
  - share/rbbt_commands/stat/abs
387
390
  - share/rbbt_commands/stat/boxplot
388
391
  - share/rbbt_commands/stat/compare_lists
@@ -449,6 +452,8 @@ files:
449
452
  - test/rbbt/association/test_open.rb
450
453
  - test/rbbt/association/test_util.rb
451
454
  - test/rbbt/entity/test_identifiers.rb
455
+ - test/rbbt/hpc/test_batch.rb
456
+ - test/rbbt/hpc/test_slurm.rb
452
457
  - test/rbbt/knowledge_base/test_enrichment.rb
453
458
  - test/rbbt/knowledge_base/test_entity.rb
454
459
  - test/rbbt/knowledge_base/test_query.rb
@@ -499,6 +504,7 @@ files:
499
504
  - test/rbbt/util/concurrency/test_threads.rb
500
505
  - test/rbbt/util/log/test_progress.rb
501
506
  - test/rbbt/util/misc/test_bgzf.rb
507
+ - test/rbbt/util/misc/test_development.rb
502
508
  - test/rbbt/util/misc/test_format.rb
503
509
  - test/rbbt/util/misc/test_lock.rb
504
510
  - test/rbbt/util/misc/test_multipart_payload.rb
@@ -551,7 +557,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
551
557
  - !ruby/object:Gem::Version
552
558
  version: '0'
553
559
  requirements: []
554
- rubygems_version: 3.0.6
560
+ rubygems_version: 3.1.4
555
561
  signing_key:
556
562
  specification_version: 4
557
563
  summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
@@ -567,6 +573,7 @@ test_files:
567
573
  - test/rbbt/resource/test_path.rb
568
574
  - test/rbbt/util/test_colorize.rb
569
575
  - test/rbbt/util/test_procpath.rb
576
+ - test/rbbt/util/misc/test_development.rb
570
577
  - test/rbbt/util/misc/test_omics.rb
571
578
  - test/rbbt/util/misc/test_pipes.rb
572
579
  - test/rbbt/util/misc/test_format.rb
@@ -617,6 +624,8 @@ test_files:
617
624
  - test/rbbt/tsv/parallel/test_traverse.rb
618
625
  - test/rbbt/tsv/test_stream.rb
619
626
  - test/rbbt/test_association.rb
627
+ - test/rbbt/hpc/test_batch.rb
628
+ - test/rbbt/hpc/test_slurm.rb
620
629
  - test/rbbt/association/test_database.rb
621
630
  - test/rbbt/association/test_item.rb
622
631
  - test/rbbt/association/test_open.rb