rbbt-util 5.28.14 → 5.30.0

data/share/rbbt_commands/slurm/orchestrate

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+require 'rbbt/workflow/usage'
+require 'rbbt/hpc'
+require 'rbbt/hpc/orchestrate'
+require 'time'
+
+$slurm_options = SOPT.get <<EOF
+-dr--dry_run Print only the template
+-cj--clean_job Clean job
+--drbbt* Use development version of rbbt 
+-sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
+-c--contain* Contain in directory (using Singularity)
+-s--sync* Contain in directory and sync jobs
+-e--exclusive Make exclusive use of the node
+-hm--highmem Make use of highmem cores
+-wc--wipe_container* Wipe the jobs from the contain directory
+-CS--contain_and_sync Contain and sync to default locations
+-ci--copy_image When using a container directory, copy image there
+-t--tail Tail the logs
+-SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
+-q--queue* Queue
+-t--task_cpus* Tasks
+-W--workflows* Additional workflows
+-tm--time* Time
+-OR--orchestration_rules* Orchestration rules
+-rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
+EOF
+
+class Step
+    def run(*args)
+        if done?
+            self.load
+        else
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::SLURM.orchestrate_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
+        end
+    end
+end
+
+ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
+load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/{workflow/slurm → slurm/task}

@@ -9,8 +9,9 @@ require 'time'
 $slurm_options = SOPT.get <<EOF
 -dr--dry_run Print only the template
 -cj--clean_job Clean job
---drbbt Use development version of rbbt 
+--drbbt* Use development version of rbbt 
 -sing--singularity Use Singularity
+-ug--user_group* Use alternative user group for group project directory
 -c--contain* Contain in directory (using Singularity)
 -s--sync* Contain in directory and sync jobs
 -e--exclusive Make exclusive use of the node
@@ -19,10 +20,12 @@ $slurm_options = SOPT.get <<EOF
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
+-SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
+-W--workflows* Additional workflows
 -tm--time* Time
--S--server* SLURM login node
+-rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF
 
 class Step
@@ -30,9 +33,14 @@ class Step
         if done?
             self.load
         else
-            Marenostrum::SLURM.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            begin
+                Log.debug "Issuing SLURM job for #{self.path}"
+                HPC::SLURM.run_job(self, SOPT::GOT_OPTIONS.merge($slurm_options)) 
+            rescue HPC::SBATCH
+            end
         end
     end
 end
 
+ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/tsv/slice

@@ -35,7 +35,7 @@ file = case file
 fields = options[:fields]
 raise ParameterException, "Please specify the fields to slice" if fields.nil?
 
-options[:header_hash] = options["header_hash"]
+options[:header_hash] ||= options["header_hash"]
 
 case
 when options[:tokyocabinet]
@@ -45,8 +45,8 @@ when options[:tokyocabinet_bd]
   tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
   puts tsv.summary
 else
-  stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :keys => fields, :unnamed => true) do |*p|
-    p * "\t"
+  stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :fields => fields.split(","), :unnamed => true) do |k,fields,names|
+    [k,fields].flatten * "\t"
   end
   puts stream.read
   exit 0

data/share/rbbt_commands/workflow/info

@@ -86,7 +86,7 @@ messages = info[:messages]
 backtrace = info[:backtrace]
 pid = info[:pid]
 exception = info[:exception]
-rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :pid, :archived_info]
+rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :archived_info]
 
 
 puts Log.color(:magenta, "File") << ": " << step.path

data/share/rbbt_commands/workflow/task

@@ -20,7 +20,7 @@ def usage(workflow = nil, task = nil, exception=nil, abridge = false)
   puts
   if workflow.nil?
     puts "No workflow specified. Use `rbbt workflow list` to list available workflows."
-    exit -1 
+    exit! -1 
   end
 
   if task.nil?
@@ -203,10 +203,11 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 -prec--prepare_cpus* Number of dependencies prepared in parallel
 -rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
 -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
+-PERF--procpath_performance* Measure performance using procpath
 EOF
 
 workflow = ARGV.shift
-usage and exit -1 if workflow.nil?
+usage and exit! -1 if workflow.nil?
 
 task     = ARGV.shift
 
@@ -232,7 +233,8 @@ else
   remote_workflows = {}
 end
 
-Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
+#Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
+Workflow.workdir.search_paths.merge!({:workdir => File.expand_path(options.delete(:workdir_all)), :default => :workdir }) if options[:workdir_all]
 
 workflow = Workflow.require_workflow workflow
 
@@ -406,6 +408,13 @@ begin
     exit 0
   end
 
+  if options[:procpath_performance]
+    require 'rbbt/util/procpath'
+    job.fork
+    job.soft_grace
+    pid = job.info[:pid]
+    ProcPath.monitor(pid, options[:procpath_performance])
+  end
 
   if do_fork
     ENV["RBBT_NO_PROGRESS"] = "true"
@@ -422,7 +431,6 @@ begin
     res = job
   end
 
-
   if options.delete(:printpath)
     job.join
     raise job.messages.last if (job.error? || job.aborted?) && job.messages
@@ -486,7 +494,7 @@ rescue ParameterException
   puts
   report_options saved_job_options
   puts
-  exit -1
+  exit! -1
 end
 
 if options.delete(:list_job_files)
@@ -538,7 +546,7 @@ when Step
         io.abort if io.respond_to? :abort
         io.join  if io.respond_to? :join
       ensure
-        exit -1
+        exit! -1
       end
     rescue Exception
       Log.exception $!
@@ -547,9 +555,11 @@ when Step
         io.abort if io.respond_to? :abort
         io.join  if io.respond_to? :join
       ensure
-        exit -1
+        exit! -1
       end
     end
+  elsif detach
+    exit! 0
   else
     res.join
     out.puts Open.read(res.path) if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)

data/share/rbbt_commands/workflow/write_info

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/workflow'
+
+require 'rbbt-util'
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Examine the info of a job result
+
+$ rbbt workflow info <job-result> <key> <value>
+
+-h--help Help
+-f--force Write info even if key is already present
+-r--recursive Write info for all dependencies as well
+-p--check_pid Check that recursive jobs where created by the same process
+EOF
+
+SOPT.usage if options[:help]
+
+file, key, value = ARGV
+
+force, recursive, check_pid = options.values_at :force, :recursive, :check_pid
+
+def get_step(file)
+  file = file.sub(/\.(info|files)/,'')
+  step = Workflow.load_step file
+  step
+end
+
+raise ParameterException if key.nil? || value.nil?
+
+if %w(DELETE nil).include? value
+  value = nil
+  force = true
+end
+
+step = get_step file
+
+step.set_info key, value if force || ! step.info.include?(key)
+
+pid = step.info[:pid]
+host = step.info[:pid_hostname]
+
+step.rec_dependencies.each do |dep|
+  dep.set_info key, value if (force || ! dep.info.include?(key)) && (!check_pid || dep.info[:pid].to_s == pid and dep.info[:pid_hostname] == host)
+rescue
+  Log.warn "Could no set info #{key} for #{dep.path}: #{$!.message}"
+end if recursive

data/test/rbbt/test_workflow.rb

@@ -386,9 +386,9 @@ class TestWorkflow < Test::Unit::TestCase
     job.run
     Misc.with_env "RBBT_UPDATE", 'true' do
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
-      job = TestWF.job(:t3)
-      job.step(:t1).clean
-      assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
+      #job = TestWF.job(:t3)
+      #job.step(:t1).clean
+      #assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
       job = TestWF.job(:t3).recursive_clean
       job.run
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
@@ -427,7 +427,7 @@ class TestWorkflow < Test::Unit::TestCase
     TmpFile.with_file do |dir|
       Path.setup(dir)
       Step.save_job_inputs(job, dir)
-      assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.read']
+      assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.yaml']
       inputs  = Workflow.load_inputs(dir, [:file], :file => :file)
       assert_equal inputs, {:file => 'code'}
     end
@@ -455,9 +455,9 @@ class TestWorkflow < Test::Unit::TestCase
     job.run
     Misc.with_env "RBBT_UPDATE", 'true' do
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
-      job = TestWF.job(:t3)
-      job.step(:t1).clean
-      assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
+      #job = TestWF.job(:t3)
+      #job.step(:t1).clean
+      #assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
       job = TestWF.job(:t3).recursive_clean
       job.run
       assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?

data/test/rbbt/util/test_procpath.rb

@@ -0,0 +1,23 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/util/procpath'
+
+class TestProcPath < Test::Unit::TestCase
+  def test_record_and_plot
+    Log.with_severity 0 do
+      pid = Process.fork do
+        a = ""
+        (0..1000).each do 
+          a << (0..rand(10000).to_i).to_a.collect{|i| "TEST #{i}" } * " "
+          sleep 0.1
+        end
+      end
+
+      TmpFile.with_file(nil, false) do |db|
+
+        ProcPath.record(pid, db, :interval => '1', "recnum" => 100)
+        ProcPath.plot(db, db + '.svg', "moving-average-window" => 1 )
+      end
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.28.14
+  version: 5.30.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-18 00:00:00.000000000 Z
+date: 2021-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -211,6 +211,8 @@ files:
 - lib/rbbt/entity/identifiers.rb
 - lib/rbbt/fix_width_table.rb
 - lib/rbbt/hpc.rb
+- lib/rbbt/hpc/orchestrate.rb
+- lib/rbbt/hpc/slurm.rb
 - lib/rbbt/knowledge_base.rb
 - lib/rbbt/knowledge_base/enrichment.rb
 - lib/rbbt/knowledge_base/entity.rb
@@ -298,6 +300,7 @@ files:
 - lib/rbbt/util/misc/system.rb
 - lib/rbbt/util/named_array.rb
 - lib/rbbt/util/open.rb
+- lib/rbbt/util/procpath.rb
 - lib/rbbt/util/python.rb
 - lib/rbbt/util/semaphore.rb
 - lib/rbbt/util/simpleDSL.rb
@@ -376,6 +379,10 @@ files:
 - share/rbbt_commands/resource/produce
 - share/rbbt_commands/resource/read
 - share/rbbt_commands/rsync
+- share/rbbt_commands/slurm/clean
+- share/rbbt_commands/slurm/list
+- share/rbbt_commands/slurm/orchestrate
+- share/rbbt_commands/slurm/task
 - share/rbbt_commands/stat/abs
 - share/rbbt_commands/stat/boxplot
 - share/rbbt_commands/stat/compare_lists
@@ -430,9 +437,9 @@ files:
 - share/rbbt_commands/workflow/remote/list
 - share/rbbt_commands/workflow/remote/remove
 - share/rbbt_commands/workflow/server
-- share/rbbt_commands/workflow/slurm
 - share/rbbt_commands/workflow/task
 - share/rbbt_commands/workflow/trace
+- share/rbbt_commands/workflow/write_info
 - share/unicorn.rb
 - share/workflow_config.ru
 - test/rbbt/annotations/test_util.rb
@@ -511,6 +518,7 @@ files:
 - test/rbbt/util/test_log.rb
 - test/rbbt/util/test_misc.rb
 - test/rbbt/util/test_open.rb
+- test/rbbt/util/test_procpath.rb
 - test/rbbt/util/test_python.rb
 - test/rbbt/util/test_semaphore.rb
 - test/rbbt/util/test_simpleDSL.rb
@@ -558,6 +566,7 @@ test_files:
 - test/rbbt/workflow/test_task.rb
 - test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb
+- test/rbbt/util/test_procpath.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/misc/test_format.rb