rbbt-util 5.30.10 → 5.31.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/lib/rbbt/hpc.rb +3 -0
  3. data/lib/rbbt/hpc/batch.rb +621 -0
  4. data/lib/rbbt/hpc/lsf.rb +119 -0
  5. data/lib/rbbt/hpc/orchestrate.rb +13 -12
  6. data/lib/rbbt/hpc/slurm.rb +62 -559
  7. data/lib/rbbt/resource/path.rb +3 -1
  8. data/lib/rbbt/tsv/accessor.rb +5 -2
  9. data/lib/rbbt/tsv/dumper.rb +1 -0
  10. data/lib/rbbt/tsv/parallel/traverse.rb +1 -1
  11. data/lib/rbbt/tsv/stream.rb +5 -6
  12. data/lib/rbbt/util/cmd.rb +15 -1
  13. data/lib/rbbt/util/log.rb +22 -1
  14. data/lib/rbbt/util/log/progress.rb +17 -2
  15. data/lib/rbbt/util/log/progress/report.rb +36 -3
  16. data/lib/rbbt/util/misc/development.rb +2 -2
  17. data/lib/rbbt/util/misc/inspect.rb +17 -1
  18. data/lib/rbbt/util/misc/omics.rb +60 -1
  19. data/lib/rbbt/util/misc/options.rb +5 -0
  20. data/lib/rbbt/workflow/accessor.rb +7 -2
  21. data/lib/rbbt/workflow/definition.rb +2 -2
  22. data/lib/rbbt/workflow/step/accessor.rb +10 -6
  23. data/lib/rbbt/workflow/step/run.rb +9 -0
  24. data/lib/rbbt/workflow/usage.rb +13 -13
  25. data/lib/rbbt/workflow/util/provenance.rb +26 -21
  26. data/share/config.ru +3 -3
  27. data/share/rbbt_commands/{slurm → hpc}/clean +91 -18
  28. data/share/rbbt_commands/{slurm → hpc}/list +119 -31
  29. data/share/rbbt_commands/hpc/orchestrate +81 -0
  30. data/share/rbbt_commands/hpc/tail +81 -0
  31. data/share/rbbt_commands/hpc/task +80 -0
  32. data/test/rbbt/hpc/test_batch.rb +65 -0
  33. data/test/rbbt/hpc/test_slurm.rb +30 -0
  34. data/test/rbbt/util/misc/test_development.rb +11 -0
  35. data/test/test_helper.rb +3 -1
  36. metadata +16 -7
  37. data/share/rbbt_commands/slurm/orchestrate +0 -48
  38. data/share/rbbt_commands/slurm/task +0 -46
@@ -1,48 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require 'rbbt/util/simpleopt'
4
- require 'rbbt/workflow'
5
- require 'rbbt/workflow/usage'
6
- require 'rbbt/hpc'
7
- require 'rbbt/hpc/orchestrate'
8
- require 'time'
9
-
10
- $slurm_options = SOPT.get <<EOF
11
- -dr--dry_run Print only the template
12
- -cj--clean_job Clean job
13
- --drbbt* Use development version of rbbt
14
- -sing--singularity Use Singularity
15
- -ug--user_group* Use alternative user group for group project directory
16
- -c--contain* Contain in directory (using Singularity)
17
- -s--sync* Contain in directory and sync jobs
18
- -e--exclusive Make exclusive use of the node
19
- -hm--highmem Make use of highmem cores
20
- -wc--wipe_container* Wipe the jobs from the contain directory
21
- -CS--contain_and_sync Contain and sync to default locations
22
- -ci--copy_image When using a container directory, copy image there
23
- -t--tail Tail the logs
24
- -SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
25
- -q--queue* Queue
26
- -t--task_cpus* Tasks
27
- -W--workflows* Additional workflows
28
- -tm--time* Time
29
- -OR--orchestration_rules* Orchestration rules
30
- -rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
31
- EOF
32
-
33
- class Step
34
- def run(*args)
35
- if done?
36
- self.load
37
- else
38
- begin
39
- Log.debug "Issuing SLURM job for #{self.path}"
40
- HPC::SLURM.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options))
41
- rescue HPC::SBATCH
42
- end
43
- end
44
- end
45
- end
46
-
47
- ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
48
- load Rbbt.share.rbbt_commands.workflow.task.find
@@ -1,46 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require 'rbbt/util/simpleopt'
4
- require 'rbbt/workflow'
5
- require 'rbbt/workflow/usage'
6
- require 'rbbt/hpc'
7
- require 'time'
8
-
9
- $slurm_options = SOPT.get <<EOF
10
- -dr--dry_run Print only the template
11
- -cj--clean_job Clean job
12
- --drbbt* Use development version of rbbt
13
- -sing--singularity Use Singularity
14
- -ug--user_group* Use alternative user group for group project directory
15
- -c--contain* Contain in directory (using Singularity)
16
- -s--sync* Contain in directory and sync jobs
17
- -e--exclusive Make exclusive use of the node
18
- -hm--highmem Make use of highmem cores
19
- -wc--wipe_container* Wipe the jobs from the contain directory
20
- -CS--contain_and_sync Contain and sync to default locations
21
- -ci--copy_image When using a container directory, copy image there
22
- -t--tail Tail the logs
23
- -SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
24
- -q--queue* Queue
25
- -t--task_cpus* Tasks
26
- -W--workflows* Additional workflows
27
- -tm--time* Time
28
- -rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
29
- EOF
30
-
31
- class Step
32
- def run(*args)
33
- if done?
34
- self.load
35
- else
36
- begin
37
- Log.debug "Issuing SLURM job for #{self.path}"
38
- HPC::SLURM.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options))
39
- rescue HPC::SBATCH
40
- end
41
- end
42
- end
43
- end
44
-
45
- ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
46
- load Rbbt.share.rbbt_commands.workflow.task.find