rbbt-util 5.29.1 → 5.30.2

data/share/rbbt_commands/slurm/orchestrate

@@ -21,11 +21,12 @@ $slurm_options = SOPT.get <<EOF
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
+-SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
 -W--workflows* Additional workflows
 -tm--time* Time
--R--rules* Orchestration rules
+-OR--orchestration_rules* Orchestration rules
 -rmb--remove_slurm_basedir Remove the SLURM working directory (command, STDIN, exit status, ...)
 EOF
 
@@ -43,5 +44,5 @@ class Step
     end
 end
 
-ARGV.concat ["-W", $slurm_options[:workflows]] if $slurm_options[:workflows]
+ARGV.concat ["-W", $slurm_options[:workflows], '--detach'] if $slurm_options[:workflows]
 load Rbbt.share.rbbt_commands.workflow.task.find

data/share/rbbt_commands/slurm/task

@@ -20,6 +20,7 @@ $slurm_options = SOPT.get <<EOF
 -CS--contain_and_sync Contain and sync to default locations
 -ci--copy_image When using a container directory, copy image there
 -t--tail Tail the logs
+-SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
 -q--queue* Queue
 -t--task_cpus* Tasks
 -W--workflows* Additional workflows

data/share/rbbt_commands/tsv/slice

@@ -35,7 +35,7 @@ file = case file
 fields = options[:fields]
 raise ParameterException, "Please specify the fields to slice" if fields.nil?
 
-options[:header_hash] = options["header_hash"]
+options[:header_hash] ||= options["header_hash"]
 
 case
 when options[:tokyocabinet]
@@ -45,8 +45,8 @@ when options[:tokyocabinet_bd]
   tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
   puts tsv.summary
 else
-  stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :keys => fields, :unnamed => true) do |*p|
-    p * "\t"
+  stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :fields => fields.split(","), :unnamed => true) do |k,fields,names|
+    [k,fields].flatten * "\t"
   end
   puts stream.read
   exit 0

data/share/rbbt_commands/workflow/info

@@ -86,7 +86,7 @@ messages = info[:messages]
 backtrace = info[:backtrace]
 pid = info[:pid]
 exception = info[:exception]
-rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :pid, :archived_info]
+rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :archived_info]
 
 
 puts Log.color(:magenta, "File") << ": " << step.path

data/share/rbbt_commands/workflow/task

@@ -20,7 +20,7 @@ def usage(workflow = nil, task = nil, exception=nil, abridge = false)
   puts
   if workflow.nil?
     puts "No workflow specified. Use `rbbt workflow list` to list available workflows."
-    exit -1 
+    exit! -1 
   end
 
   if task.nil?
@@ -203,10 +203,11 @@ The `recursive_clean` cleans all the job dependency steps recursively.
 -prec--prepare_cpus* Number of dependencies prepared in parallel
 -rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
 -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
+-PERF--procpath_performance* Measure performance using procpath
 EOF
 
 workflow = ARGV.shift
-usage and exit -1 if workflow.nil?
+usage and exit! -1 if workflow.nil?
 
 task     = ARGV.shift
 
@@ -232,7 +233,8 @@ else
   remote_workflows = {}
 end
 
-Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
+#Workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir_all))) if options[:workdir_all]
+Workflow.workdir.search_paths.merge!({:workdir => File.expand_path(options.delete(:workdir_all)), :default => :workdir }) if options[:workdir_all]
 
 workflow = Workflow.require_workflow workflow
 
@@ -406,6 +408,23 @@ begin
     exit 0
   end
 
+  if options[:procpath_performance]
+    require 'rbbt/util/procpath'
+    current_pid = job.info[:pid]
+    job.fork
+    job.soft_grace
+    sleep 2 if job.info[:pid] == current_pid
+    if job.info[:pid] != current_pid
+      pid = job.info[:pid]
+      begin
+        ProcPath.monitor(pid, options[:procpath_performance])
+      rescue Errno::ECHILD
+        Log.warn "Procpath didn't find process #{pid} to monitor. Maybe it finished already"
+      rescue
+        Log.warn "Procpath failed: #{$!.message}"
+      end
+    end
+  end
 
   if do_fork
     ENV["RBBT_NO_PROGRESS"] = "true"
@@ -422,7 +441,6 @@ begin
     res = job
   end
 
-
   if options.delete(:printpath)
     job.join
     raise job.messages.last if (job.error? || job.aborted?) && job.messages
@@ -486,7 +504,7 @@ rescue ParameterException
   puts
   report_options saved_job_options
   puts
-  exit -1
+  exit! -1
 end
 
 if options.delete(:list_job_files)
@@ -538,7 +556,7 @@ when Step
         io.abort if io.respond_to? :abort
         io.join  if io.respond_to? :join
       ensure
-        exit -1
+        exit! -1
       end
     rescue Exception
       Log.exception $!
@@ -547,9 +565,11 @@ when Step
         io.abort if io.respond_to? :abort
         io.join  if io.respond_to? :join
       ensure
-        exit -1
+        exit! -1
       end
     end
+  elsif detach
+    exit! 0
   else
     res.join
     out.puts Open.read(res.path) if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)

data/share/rbbt_commands/workflow/write_info

@@ -0,0 +1,52 @@
+#!/usr/bin/env ruby
+
+require 'rbbt/workflow'
+
+require 'rbbt-util'
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+Examine the info of a job result
+
+$ rbbt workflow info <job-result> <key> <value>
+
+-h--help Help
+-f--force Write info even if key is already present
+-r--recursive Write info for all dependencies as well
+-p--check_pid Check that recursive jobs where created by the same process
+EOF
+
+SOPT.usage if options[:help]
+
+file, key, value = ARGV
+
+force, recursive, check_pid = options.values_at :force, :recursive, :check_pid
+
+def get_step(file)
+  file = file.sub(/\.(info|files)/,'')
+  step = Workflow.load_step file
+  step
+end
+
+raise ParameterException if key.nil? || value.nil?
+
+if %w(DELETE nil).include? value
+  value = nil
+  force = true
+end
+
+step = get_step file
+
+step.set_info key, value if force || ! step.info.include?(key)
+
+pid = step.info[:pid]
+host = step.info[:pid_hostname]
+
+step.rec_dependencies.each do |dep|
+  dep.set_info key, value if (force || ! dep.info.include?(key)) && (!check_pid || dep.info[:pid].to_s == pid and dep.info[:pid_hostname] == host)
+rescue
+  Log.warn "Could no set info #{key} for #{dep.path}: #{$!.message}"
+end if recursive

data/test/rbbt/util/test_procpath.rb

@@ -0,0 +1,23 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/util/procpath'
+
+class TestProcPath < Test::Unit::TestCase
+  def test_record_and_plot
+    Log.with_severity 0 do
+      pid = Process.fork do
+        a = ""
+        (0..1000).each do 
+          a << (0..rand(10000).to_i).to_a.collect{|i| "TEST #{i}" } * " "
+          sleep 0.1
+        end
+      end
+
+      TmpFile.with_file(nil, false) do |db|
+
+        ProcPath.record(pid, db, :interval => '1', "recnum" => 100)
+        ProcPath.plot(db, db + '.svg', "moving-average-window" => 1 )
+      end
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.29.1
+  version: 5.30.2
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-01-14 00:00:00.000000000 Z
+date: 2021-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -300,6 +300,7 @@ files:
 - lib/rbbt/util/misc/system.rb
 - lib/rbbt/util/named_array.rb
 - lib/rbbt/util/open.rb
+- lib/rbbt/util/procpath.rb
 - lib/rbbt/util/python.rb
 - lib/rbbt/util/semaphore.rb
 - lib/rbbt/util/simpleDSL.rb
@@ -378,6 +379,7 @@ files:
 - share/rbbt_commands/resource/produce
 - share/rbbt_commands/resource/read
 - share/rbbt_commands/rsync
+- share/rbbt_commands/slurm/clean
 - share/rbbt_commands/slurm/list
 - share/rbbt_commands/slurm/orchestrate
 - share/rbbt_commands/slurm/task
@@ -437,6 +439,7 @@ files:
 - share/rbbt_commands/workflow/server
 - share/rbbt_commands/workflow/task
 - share/rbbt_commands/workflow/trace
+- share/rbbt_commands/workflow/write_info
 - share/unicorn.rb
 - share/workflow_config.ru
 - test/rbbt/annotations/test_util.rb
@@ -515,6 +518,7 @@ files:
 - test/rbbt/util/test_log.rb
 - test/rbbt/util/test_misc.rb
 - test/rbbt/util/test_open.rb
+- test/rbbt/util/test_procpath.rb
 - test/rbbt/util/test_python.rb
 - test/rbbt/util/test_semaphore.rb
 - test/rbbt/util/test_simpleDSL.rb
@@ -562,6 +566,7 @@ test_files:
 - test/rbbt/workflow/test_task.rb
 - test/rbbt/resource/test_path.rb
 - test/rbbt/util/test_colorize.rb
+- test/rbbt/util/test_procpath.rb
 - test/rbbt/util/misc/test_omics.rb
 - test/rbbt/util/misc/test_pipes.rb
 - test/rbbt/util/misc/test_format.rb