rbbt-util 5.30.13 → 5.31.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (29) hide show
  1. checksums.yaml +4 -4
  2. data/lib/rbbt/hpc.rb +3 -0
  3. data/lib/rbbt/hpc/batch.rb +623 -0
  4. data/lib/rbbt/hpc/lsf.rb +119 -0
  5. data/lib/rbbt/hpc/orchestrate.rb +12 -11
  6. data/lib/rbbt/hpc/slurm.rb +62 -567
  7. data/lib/rbbt/resource/path.rb +3 -1
  8. data/lib/rbbt/tsv/accessor.rb +5 -2
  9. data/lib/rbbt/tsv/dumper.rb +1 -0
  10. data/lib/rbbt/tsv/parallel/traverse.rb +1 -1
  11. data/lib/rbbt/tsv/stream.rb +5 -6
  12. data/lib/rbbt/util/log.rb +22 -1
  13. data/lib/rbbt/util/misc/development.rb +2 -2
  14. data/lib/rbbt/util/misc/options.rb +5 -0
  15. data/lib/rbbt/workflow/step/accessor.rb +1 -1
  16. data/lib/rbbt/workflow/usage.rb +13 -13
  17. data/share/config.ru +3 -3
  18. data/share/rbbt_commands/{slurm → hpc}/clean +91 -18
  19. data/share/rbbt_commands/{slurm → hpc}/list +100 -30
  20. data/share/rbbt_commands/hpc/orchestrate +81 -0
  21. data/share/rbbt_commands/hpc/tail +81 -0
  22. data/share/rbbt_commands/hpc/task +80 -0
  23. data/test/rbbt/hpc/test_batch.rb +65 -0
  24. data/test/rbbt/hpc/test_slurm.rb +30 -0
  25. data/test/rbbt/util/misc/test_development.rb +11 -0
  26. data/test/test_helper.rb +3 -1
  27. metadata +16 -7
  28. data/share/rbbt_commands/slurm/orchestrate +0 -48
  29. data/share/rbbt_commands/slurm/task +0 -46
data/share/rbbt_commands/hpc/orchestrate ADDED
@@ -0,0 +1,81 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rbbt/util/simpleopt'
4
+ require 'rbbt/workflow'
5
+ require 'rbbt/workflow/usage'
6
+ require 'rbbt/hpc'
7
+ require 'rbbt/hpc/orchestrate'
8
+ require 'time'
9
+
10
+ $slurm_options = SOPT.get <<EOF
11
+ -dr--dry_run Print only the template
12
+ -cj--clean_job Clean job
13
+ --drbbt* Use development version of rbbt
14
+ -sing--singularity Use Singularity
15
+ -ug--user_group* Use alternative user group for group project directory
16
+ -c--contain* Contain in directory (using Singularity)
17
+ -s--sync* Contain in directory and sync jobs
18
+ -e--exclusive Make exclusive use of the node
19
+ -hm--highmem Make use of highmem cores
20
+ -wc--wipe_container* Wipe the jobs from the contain directory
21
+ -CS--contain_and_sync Contain and sync to default locations
22
+ -ci--copy_image When using a container directory, copy image there
23
+ -t--tail Tail the logs
24
+ -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
25
+ -q--queue* Queue
26
+ -t--task_cpus* Tasks
27
+ -W--workflows* Additional workflows
28
+ -tm--time* Time
29
+ -OR--orchestration_rules* Orchestration rules
30
+ -rmb--remove_batch_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
31
+ EOF
32
+
33
+ batch_system = $slurm_options.delete :batch_system
34
+ batch_system ||= 'auto'
35
+
36
+ HPC::BATCH_MODULE = case batch_system.to_s.downcase
37
+ when 'slurm'
38
+ HPC::SLURM
39
+ when 'lsf'
40
+ HPC::LSF
41
+ when 'auto'
42
+ case $previous_commands.last
43
+ when 'slurm'
44
+ HPC::SLURM
45
+ when 'lsf'
46
+ HPC::LSF
47
+ else
48
+ case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
49
+ when 'slurm'
50
+ HPC::SLURM
51
+ when 'lsf'
52
+ HPC::LSF
53
+ else
54
+ case ENV["BATCH_SYSTEM"].to_s.downcase
55
+ when 'slurm'
56
+ HPC::SLURM
57
+ when 'lsf'
58
+ HPC::LSF
59
+ end
60
+ end
61
+ end
62
+ end
63
+
64
+ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
65
+
66
+ class Step
67
+ def run(*args)
68
+ if done?
69
+ self.load
70
+ else
71
+ begin
72
+ Log.debug "Issuing SLURM job for #{self.path}"
73
+ HPC::BATCH_MODULE.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options))
74
+ rescue HPC::SBATCH
75
+ end
76
+ end
77
+ end
78
+ end
79
+
80
+ ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
81
+ load Rbbt.share.rbbt_commands.workflow.task.find
data/share/rbbt_commands/hpc/tail ADDED
@@ -0,0 +1,81 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rbbt-util'
4
+ require 'rbbt/util/simpleopt'
5
+ require 'rbbt/hpc'
6
+
7
+ #$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
8
+
9
+ options = SOPT.setup <<EOF
10
+
11
+ Queue a job in Marenostrum
12
+
13
+ $ rbbt slurm tail <directory> [options]
14
+
15
+ -h--help Print this help
16
+ -d--done Done jobs only
17
+ -e--error Error jobs only
18
+ -a--aborted SLURM aboted jobs
19
+ -r--running Running jobs only
20
+ -q--queued Queued jobs only
21
+ -j--job* Job ids
22
+ -s--search* Regular expression
23
+ -t--tail* Show the last lines of the STDERR
24
+ -p--progress Report progress of job and the dependencies
25
+ -SBP--sbatch_parameters show sbatch parameters
26
+ -PERF--procpath_performance show Procpath performance summary
27
+ -sacct--sacct_peformance show sacct performance summary
28
+ -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
29
+ EOF
30
+
31
+ if options[:help]
32
+ if defined? rbbt_usage
33
+ rbbt_usage
34
+ else
35
+ puts SOPT.doc
36
+ end
37
+ exit 0
38
+ end
39
+
40
+ batch_system = options.delete :batch_system
41
+ batch_system ||= 'auto'
42
+
43
+ HPC::BATCH_MODULE = case batch_system.to_s.downcase
44
+ when 'slurm'
45
+ HPC::SLURM
46
+ when 'lsf'
47
+ HPC::LSF
48
+ when 'auto'
49
+ case $previous_commands.last
50
+ when 'slurm'
51
+ HPC::SLURM
52
+ when 'lsf'
53
+ HPC::LSF
54
+ else
55
+ case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
56
+ when 'slurm'
57
+ HPC::SLURM
58
+ when 'lsf'
59
+ HPC::LSF
60
+ else
61
+ case ENV["BATCH_SYSTEM"].to_s.downcase
62
+ when 'slurm'
63
+ HPC::SLURM
64
+ when 'lsf'
65
+ HPC::LSF
66
+ end
67
+ end
68
+ end
69
+ end
70
+
71
+ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
72
+
73
+ directory = ARGV.shift
74
+
75
+ raise ParameterException if directory.nil?
76
+
77
+ directory = File.dirname(directory) unless File.directory?(directory)
78
+
79
+ require 'rbbt/hpc/slurm'
80
+
81
+ HPC::BATCH_MODULE.follow_job directory, true
data/share/rbbt_commands/hpc/task ADDED
@@ -0,0 +1,80 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rbbt/util/simpleopt'
4
+ require 'rbbt/workflow'
5
+ require 'rbbt/workflow/usage'
6
+ require 'rbbt/hpc'
7
+ require 'time'
8
+
9
+ $slurm_options = SOPT.get <<EOF
10
+ -dr--dry_run Print only the template
11
+ -cj--clean_job Clean job
12
+ --drbbt* Use development version of rbbt
13
+ -sing--singularity Use Singularity
14
+ -ug--user_group* Use alternative user group for group project directory
15
+ -c--contain* Contain in directory (using Singularity)
16
+ -s--sync* Contain in directory and sync jobs
17
+ -e--exclusive Make exclusive use of the node
18
+ -hm--highmem Make use of highmem cores
19
+ -wc--wipe_container* Wipe the jobs from the contain directory
20
+ -CS--contain_and_sync Contain and sync to default locations
21
+ -ci--copy_image When using a container directory, copy image there
22
+ -t--tail Tail the logs
23
+ -BPP--batch_procpath* Save Procpath performance for batch job; specify only options
24
+ -q--queue* Queue
25
+ -t--task_cpus* Tasks
26
+ -W--workflows* Additional workflows
27
+ -tm--time* Time
28
+ -rmb--remove_batch_dir Remove the batch working directory (command, STDIN, exit status, ...)
29
+ -bs--batch_system* Batch system to use: auto, lsf, slurm (default is auto-detect)
30
+ EOF
31
+
32
+ batch_system = $slurm_options.delete :batch_system
33
+ batch_system ||= 'auto'
34
+
35
+ HPC::BATCH_MODULE = case batch_system.to_s.downcase
36
+ when 'slurm'
37
+ HPC::SLURM
38
+ when 'lsf'
39
+ HPC::LSF
40
+ when 'auto'
41
+ case $previous_commands.last
42
+ when 'slurm'
43
+ HPC::SLURM
44
+ when 'lsf'
45
+ HPC::LSF
46
+ else
47
+ case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
48
+ when 'slurm'
49
+ HPC::SLURM
50
+ when 'lsf'
51
+ HPC::LSF
52
+ else
53
+ case ENV["BATCH_SYSTEM"].to_s.downcase
54
+ when 'slurm'
55
+ HPC::SLURM
56
+ when 'lsf'
57
+ HPC::LSF
58
+ end
59
+ end
60
+ end
61
+ end
62
+
63
+ raise ParameterException.new("Could not detect batch_system: #{Misc.fingerprint batch_system}") if HPC::BATCH_MODULE.nil?
64
+
65
+ class Step
66
+ def run(*args)
67
+ if done?
68
+ self.load
69
+ else
70
+ begin
71
+ Log.debug "Issuing SLURM job for #{self.path}"
72
+ HPC::BATCH_MODULE.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options))
73
+ rescue HPC::SBATCH
74
+ end
75
+ end
76
+ end
77
+ end
78
+
79
+ ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
80
+ load Rbbt.share.rbbt_commands.workflow.task.find
data/test/rbbt/hpc/test_batch.rb ADDED
@@ -0,0 +1,65 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
+ require 'rbbt/hpc/batch'
3
+ require 'rbbt/workflow'
4
+
5
+ Workflow.require_workflow "Sample"
6
+ Workflow.require_workflow "HTS"
7
+ class TestSLURM < Test::Unit::TestCase
8
+
9
+ def _test_batch_options
10
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
11
+
12
+ TmpFile.with_file do |batch_dir|
13
+
14
+ options = HPC::BATCH.batch_options(job, :batch_dir => batch_dir, :batch_modules => 'java')
15
+
16
+ iii options
17
+ end
18
+ end
19
+
20
+ def _test_template
21
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
22
+
23
+ TmpFile.with_file do |batch_dir|
24
+
25
+ template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java')
26
+ ppp template
27
+
28
+ end
29
+ end
30
+
31
+ def _test_template_singularity
32
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
33
+
34
+ TmpFile.with_file do |batch_dir|
35
+
36
+ template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :singularity => true)
37
+ ppp template
38
+
39
+ end
40
+ end
41
+
42
+ def _test_template_contain
43
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
44
+
45
+ TmpFile.with_file do |batch_dir|
46
+
47
+ template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :contain_and_sync => true, :wipe_container => 'force')
48
+ ppp template
49
+
50
+ end
51
+ end
52
+
53
+ def test_template_singularity_contain
54
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
55
+
56
+ TmpFile.with_file do |batch_dir|
57
+
58
+ template = HPC::BATCH.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java', :contain_and_sync => true, :wipe_container => 'force', :singularity => true)
59
+ ppp template
60
+
61
+ end
62
+ end
63
+
64
+ end
65
+
data/test/rbbt/hpc/test_slurm.rb ADDED
@@ -0,0 +1,30 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
+ require 'rbbt/hpc/slurm'
3
+ require 'rbbt/workflow'
4
+
5
+ Workflow.require_workflow "Sample"
6
+ Workflow.require_workflow "HTS"
7
+ class TestSLURM < Test::Unit::TestCase
8
+
9
+ def _test_template
10
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
11
+
12
+ TmpFile.with_file do |batch_dir|
13
+
14
+ template = HPC::SLURM.job_template(job, :batch_dir => batch_dir, :batch_modules => 'java')
15
+ ppp template
16
+
17
+ end
18
+ end
19
+
20
+ def test_run_job
21
+ job = Sample.job(:mutect2, "small", :reference => "hg38")
22
+
23
+ job.clean
24
+
25
+ jobid = HPC::SLURM.run_job(job, :workflows => "HTS", :batch_modules => 'java', :env_cmd => '_JAVA_OPTIONS="-Xms1g -Xmx${MAX_MEMORY}m"', :queue => :debug, :time => '01:00:00', :config_keys => "HTS_light", :task_cpus => '10', :tail => true, :clean_task => "HTS#mutect2")
26
+ iii jobid
27
+ end
28
+
29
+ end
30
+
data/test/rbbt/util/misc/test_development.rb ADDED
@@ -0,0 +1,11 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
2
+ require 'rbbt/util/misc/development'
3
+
4
+ class TestMiscDevelopment < Test::Unit::TestCase
5
+ def test_ssh
6
+ Misc.ssh_run 'mn1' do
7
+ puts "hola"
8
+ end
9
+ end
10
+ end
11
+
data/test/test_helper.rb CHANGED
@@ -20,7 +20,9 @@ class Test::Unit::TestCase
20
20
  def setup
21
21
  Random.new
22
22
 
23
- Persist.cachedir = Rbbt.tmp.test.persistence.find :user if defined? Persist
23
+ if defined? Persist
24
+ Persist.cachedir = Rbbt.tmp.test.persistence.find(:user)
25
+ end
24
26
 
25
27
  Entity.entity_property_cache = Rbbt.tmp.test.entity_property.find(:user) if defined? Entity
26
28
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.30.13
4
+ version: 5.31.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-02-21 00:00:00.000000000 Z
11
+ date: 2021-03-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -211,6 +211,8 @@ files:
211
211
  - lib/rbbt/entity/identifiers.rb
212
212
  - lib/rbbt/fix_width_table.rb
213
213
  - lib/rbbt/hpc.rb
214
+ - lib/rbbt/hpc/batch.rb
215
+ - lib/rbbt/hpc/lsf.rb
214
216
  - lib/rbbt/hpc/orchestrate.rb
215
217
  - lib/rbbt/hpc/slurm.rb
216
218
  - lib/rbbt/knowledge_base.rb
@@ -369,6 +371,11 @@ files:
369
371
  - share/rbbt_commands/file_server/add
370
372
  - share/rbbt_commands/file_server/list
371
373
  - share/rbbt_commands/file_server/remove
374
+ - share/rbbt_commands/hpc/clean
375
+ - share/rbbt_commands/hpc/list
376
+ - share/rbbt_commands/hpc/orchestrate
377
+ - share/rbbt_commands/hpc/tail
378
+ - share/rbbt_commands/hpc/task
372
379
  - share/rbbt_commands/log
373
380
  - share/rbbt_commands/migrate
374
381
  - share/rbbt_commands/migrate_job
@@ -379,10 +386,6 @@ files:
379
386
  - share/rbbt_commands/resource/produce
380
387
  - share/rbbt_commands/resource/read
381
388
  - share/rbbt_commands/rsync
382
- - share/rbbt_commands/slurm/clean
383
- - share/rbbt_commands/slurm/list
384
- - share/rbbt_commands/slurm/orchestrate
385
- - share/rbbt_commands/slurm/task
386
389
  - share/rbbt_commands/stat/abs
387
390
  - share/rbbt_commands/stat/boxplot
388
391
  - share/rbbt_commands/stat/compare_lists
@@ -449,6 +452,8 @@ files:
449
452
  - test/rbbt/association/test_open.rb
450
453
  - test/rbbt/association/test_util.rb
451
454
  - test/rbbt/entity/test_identifiers.rb
455
+ - test/rbbt/hpc/test_batch.rb
456
+ - test/rbbt/hpc/test_slurm.rb
452
457
  - test/rbbt/knowledge_base/test_enrichment.rb
453
458
  - test/rbbt/knowledge_base/test_entity.rb
454
459
  - test/rbbt/knowledge_base/test_query.rb
@@ -499,6 +504,7 @@ files:
499
504
  - test/rbbt/util/concurrency/test_threads.rb
500
505
  - test/rbbt/util/log/test_progress.rb
501
506
  - test/rbbt/util/misc/test_bgzf.rb
507
+ - test/rbbt/util/misc/test_development.rb
502
508
  - test/rbbt/util/misc/test_format.rb
503
509
  - test/rbbt/util/misc/test_lock.rb
504
510
  - test/rbbt/util/misc/test_multipart_payload.rb
@@ -551,7 +557,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
551
557
  - !ruby/object:Gem::Version
552
558
  version: '0'
553
559
  requirements: []
554
- rubygems_version: 3.0.6
560
+ rubygems_version: 3.1.4
555
561
  signing_key:
556
562
  specification_version: 4
557
563
  summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
@@ -567,6 +573,7 @@ test_files:
567
573
  - test/rbbt/resource/test_path.rb
568
574
  - test/rbbt/util/test_colorize.rb
569
575
  - test/rbbt/util/test_procpath.rb
576
+ - test/rbbt/util/misc/test_development.rb
570
577
  - test/rbbt/util/misc/test_omics.rb
571
578
  - test/rbbt/util/misc/test_pipes.rb
572
579
  - test/rbbt/util/misc/test_format.rb
@@ -617,6 +624,8 @@ test_files:
617
624
  - test/rbbt/tsv/parallel/test_traverse.rb
618
625
  - test/rbbt/tsv/test_stream.rb
619
626
  - test/rbbt/test_association.rb
627
+ - test/rbbt/hpc/test_batch.rb
628
+ - test/rbbt/hpc/test_slurm.rb
620
629
  - test/rbbt/association/test_database.rb
621
630
  - test/rbbt/association/test_item.rb
622
631
  - test/rbbt/association/test_open.rb