rbbt-util 5.26.139 → 5.26.140

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b6c406389324efc13d24ce50fdc67600bbe0b15925817134768b8f1d3b92dabc
-  data.tar.gz: 5a4bdab5412cfef2257dd9bc07906277c7071cd88a1560f3e8922b7c14a59abf
+  metadata.gz: 17c32c77d5d49d16533f6b7454312698a17b58f46c3b6c0fab860da939ecbacc
+  data.tar.gz: fa257ea2c8605343387cc7ac8c114cec37a5d65083886fb681d004a12c407975
 SHA512:
-  metadata.gz: 7f5159e1708f31695bc032469e9c5eec0ae15511ac7cb12f2d1976e15ecb13abf799f6c2ee43c70b1ed6a024abbe4d1196c9ec3c17006bd8e25ddaf83ca7ac8c
-  data.tar.gz: 9ffdb6cfd45cb919c8aaad99cd4f56c675dbedb8eae9371ef74bbd61598e5023468fc10460c6bf125d2ef47dacf41e8bae76b9bd7bf197da8182d3d7c8c189d1
+  metadata.gz: a4af9fec39133714dad6c3ebcb604d80e02984ba6bb3e2ced53e898e9a0624001161c8f2e7192d4ac0219994f86fcface0a934c0758368029edc0178111a2b8f
+  data.tar.gz: c27f66921cc667ae476ed3ed2b68a6914d76177258178075af8effb03fb390f6a3102742cf34aad9c0b4c89cbe64792f9ce29ad016500f7901699b9aa6fa7c67

data/lib/rbbt/hpc.rb

@@ -221,13 +221,13 @@ EOF
         if contain
           singularity_exec << %( -C -H "$CONTAINER_DIR" \
 -B /scratch/tmp \
+#{ group != user_group ? "-B /gpfs/projects/#{user_group}" : "" } \
+-B #{scratch_group_dir} \
+-B #{projects_group_dir} \
 -B "$SINGULARITY_RUBY_INLINE":"$CONTAINER_DIR/.ruby_inline":rw  \
 -B ~/git:"$CONTAINER_DIR/git":ro \
 #{Open.exists?('~/.rbbt/software/opt/')? '-B ~/.rbbt/software/opt/:"/opt/":ro' : '' } \
 -B ~/.rbbt:"$CONTAINER_DIR/home/":ro \
-#{ group != user_group ? "-B /gpfs/projects/#{user_group}" : "" } \
--B #{scratch_group_dir} \
--B #{projects_group_dir} \
 "$SINGULARITY_IMG")
           exec_cmd << ' TMPDIR="$CONTAINER_DIR/.rbbt/tmp" '
         else
@@ -341,13 +341,21 @@ EOF
 
 # Write exit status to file
 echo $exit_status > #{fexit}
-if [ $sync_es == '0' ]; then 
+EOF
+      if sync
+        coda +=<<-EOF 
+if [ "$sync_es" == '0' ]; then 
   unset sync_es
   exit $exit_status
 else
   exit $sync_es
 fi
 EOF
+      else
+        coda +=<<-EOF 
+exit $exit_status
+EOF
+      end
 
       template = [header, env, prep, run, coda] * "\n"
 

data/lib/rbbt/util/R/plot.rb

@@ -15,6 +15,8 @@ module R
       sources = [:plot, :svg, options[:source]].flatten.compact
       options.delete :source
 
+      field_classes = options[:field_classes]
+
       fast = options[:fast]
 
       if fast
@@ -31,6 +33,7 @@ module R
           break
         end
         values = [values] unless Array === values
+
         field_classes = values.collect do |v| 
           v = v.first if Array === v
           case v
@@ -49,7 +52,8 @@ module R
           else
             ":NA"
           end
-        end
+        end if field_classes.nil?
+
         if field_classes.any?
           options[:R_open] ||= "colClasses=c('character'," + field_classes * ", " + ')'
         else

data/lib/rbbt/util/log.rb

@@ -24,6 +24,15 @@ module Log
                      names
                    end
 
+  def self.terminal_width
+    80
+  end
+  
+  def self.compact
+    true
+  end
+
+
   def self.last_caller(stack)
     line = nil
     pos ||= 0

data/lib/rbbt/util/misc/format.rb

@@ -27,6 +27,7 @@ module Misc
 
   def self.format_paragraph(text, size = 80, indent = 0, offset = 0)
     i = 0
+    size = size + offset + indent
     re = /((?:\n\s*\n\s*)|(?:\n\s*(?=\*)))/
       text.split(re).collect do |paragraph|
       i += 1

data/lib/rbbt/util/simpleopt/parse.rb

@@ -23,12 +23,11 @@ module SOPT
       end
     end
 
-    while shortcuts.include?(short) and not shortcuts[short] == long
-      while shortcuts[short].index current * ""
-        next_letter = chars.shift
-        return nil if next_letter.nil?
-        current << next_letter
-      end
+    while shortcuts.include?(short) && shortcuts[short] != long 
+      next_letter = chars.shift
+      next_letter = chars.shift while %w(. - _).include?(next_letter)
+      return nil if next_letter.nil?
+      current << next_letter
       short = current * ""
     end
 

data/lib/rbbt/workflow/integration/cromwell.rb

@@ -0,0 +1,71 @@
+module Workflow
+
+  Rbbt.claim Rbbt.software.opt.jar["cromwell.jar"], :url, "https://github.com/broadinstitute/cromwell/releases/download/48/cromwell-48.jar"
+  Rbbt.claim Rbbt.software.opt.jar["wdltool.jar"], :url, "https://github.com/broadinstitute/wdltool/releases/download/0.14/wdltool-0.14.jar"
+
+  def run_cromwell(file, work_dir, options = {})
+    jar = Rbbt.software.opt.jar["cromwell.jar"].produce.find
+    CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}'", options.merge("add_option_dashes" => true))
+  end
+
+  def load_cromwell(file)
+    jar = Rbbt.software.opt.jar["wdltool.jar"].produce.find
+    inputs = JSON.load(CMD.cmd("java -jar '#{jar}' inputs '#{file}'"))
+
+    workflow_inputs = {}
+    inputs.each do |input,input_type|
+      workflow, task, input_name = input.split(".")
+      workflow_inputs[workflow] ||= {}
+
+      if input_name.nil?
+        input_name = task
+      else
+        input_name = [task, input_name] * "."
+      end
+
+      workflow_inputs[workflow][input_name] = input_type
+    end
+
+    workflow_inputs.each do |workflow,input_list|
+      input_list.each do |input_name,input_type|
+
+        input_type = input_type.split(" ").last.sub('?','')
+        input_type_fixed = case input_type
+                            when "File", "file"
+                              :file
+                            when "Int"
+                              :integer
+                            when /Array/
+                              :array
+                            else
+                              input_type.downcase.to_sym
+                            end
+
+        desc = [workflow, input_name] * "."
+        default = nil
+        input input_name, input_type_fixed, desc, default, :nofile => true
+      end
+
+      task workflow => :string do |*args|
+        cromwell = file
+        options = {}
+        Misc.in_dir(self.files_dir) do
+          options["metadata-output"] = file('metadata.json')
+          options["inputs"] = file('inputs')
+
+          cromwell_inputs = {}
+          self.inputs.to_hash.each do |input, value|
+            next if value.nil?
+            key = [workflow.to_s, input] * "."
+            cromwell_inputs[key] = value
+          end
+
+          Open.write(file('inputs'), cromwell_inputs.to_json )
+          Cromwell.run_cromwell(cromwell, self.files_dir, options)
+        end
+        Open.read(Dir.glob(File.join(files_dir, "/cromwell-executions/#{workflow}/*/call-*/execution/stdout")).first)
+      end
+
+    end
+  end
+end

data/lib/rbbt/workflow/step/run.rb

@@ -673,7 +673,7 @@ class Step
   end
 
   def grace
-    until done? or result or error? or aborted? or streaming? or waiting?
+    until done? || result || error? || aborted? || streaming? || waiting? 
       sleep 1 
     end
     self

data/lib/rbbt/workflow/usage.rb

@@ -43,28 +43,100 @@ module Task
         else
           puts "  #{Log.color :yellow, task.name.to_s}:"
         end
-        puts
+        puts unless Log.compact
         puts SOPT.input_doc(new_inputs, task.input_types, task.input_descriptions, task.input_defaults, true)
-        puts
+        puts unless Log.compact
       end
+      puts
     end
 
     puts Log.color(:magenta, "Returns: ") << Log.color(:blue, result_type.to_s) << "\n"
     puts
-    
+
     if selects.any?
       puts Log.color(:magenta, "Input select options")
       puts
       selects.collect{|p| p}.uniq.each do |input,options|
         puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| o.to_s} * ", ") << "\n"
-        puts
+        puts unless Log.compact
       end
+      puts
     end
   end
 end
 
 module Workflow
-  def doc(task = nil)
+
+  def dep_tree(name)
+    @dep_tree ||= {}
+    @dep_tree[name] ||= begin
+                        dep_tree = {}
+                        self.rec_dependencies(name).each do |dep|
+                          dep = dep.first if Array === dep && dep.length == 1
+
+                          workflow, task = case dep
+                                           when Array
+                                             dep.values_at 0, 1
+                                           when Symbol, String
+                                             [self, dep]
+                                           else
+                                             next
+                                           end
+
+
+                          key = [workflow, task]
+
+                          dep_tree[key] = workflow.dep_tree(task)
+                        end
+                        dep_tree
+                      end
+  end
+
+  def prov_string(tree)
+    description = ""
+
+    last = nil
+    seen = Set.new
+    tree.collect.to_a.flatten.select{|e| Symbol === e }.each do |task_name|
+
+      child = last && last.include?(task_name)
+      first = last.nil?
+      last = dep_tree(task_name).collect.to_a.flatten.select{|e| Symbol === e}
+
+      next if seen.include?(task_name)
+
+      if child
+        description << "->" << task_name.to_s
+      elsif first
+        description << "" << task_name.to_s
+      else
+        description << ";" << task_name.to_s
+      end
+    end
+    description
+  end
+
+  def prov_tree(tree, offset = 0, seen = [])
+
+    return "" if tree.empty?
+
+    lines = []
+
+    offset_str = " " * offset
+
+    lines << offset_str 
+
+    tree.each do |p,dtree| 
+      next if seen.include?(p)
+      seen.push(p)
+      workflow, task = p
+      lines << offset_str + [workflow.to_s, task.to_s] * "#" + "\n" + workflow.prov_tree(dtree, offset + 1, seen)
+    end
+
+    lines * "\n"
+  end
+
+  def doc(task = nil, abridge = false)
 
     if task.nil?
       puts Log.color :magenta, self.to_s 
@@ -88,10 +160,27 @@ module Workflow
       end
       puts
 
+      final = Set.new
+      not_final = Set.new
+      tasks.each do |name,task|
+        tree = dep_tree(name)
+        not_final += tree.keys
+        final << name unless not_final.include?(name)
+      end
+
+      not_final.each do |p|
+        final -= [p.last]
+      end
+
       tasks.each do |name,task|
         description = task.description || ""
         description = description.split("\n\n").first
-        puts Misc.format_definition_list_item(name.to_s, description, 80, 30, :yellow)
+
+        next if abridge and ! final.include?(name)
+        puts Misc.format_definition_list_item(name.to_s, description, Log.terminal_width, 20, :yellow)
+
+        prov_string = prov_string(dep_tree(name))
+        puts Log.color :blue, " ->" + prov_string if prov_string && ! prov_string.empty?
       end
 
     else
@@ -106,29 +195,46 @@ module Workflow
       #dependencies = self.rec_dependencies(task_name).collect{|dep_name| Array === dep_name ? dep_name.first.tasks[dep_name[1].to_sym] : self.tasks[dep_name.to_sym]}
       task.doc(self, self.rec_dependencies(task_name))
 
+      prov_tree = prov_tree(dep_tree(task_name))
+      if prov_tree && ! prov_tree.empty?
+
+        puts Log.color :magenta, "## DEPENDENCY GRAPH (abridged)"
+        puts
+        prov_tree.split("\n").each do |line|
+          next if line.strip.empty?
+          if m = line.match(/^( *)(\w+?)#(\w*)/i)
+              offset, workflow, task_name =  m.values_at 1, 2, 3
+            puts [offset, Log.color(:magenta, workflow), "#", Log.color(:yellow, task_name)] * ""
+          else
+            puts Log.color :blue, line 
+          end
+        end
+        puts
+      end
+
       if self.examples.include? task_name
-          self.examples[task_name].each do |example|
-
-            puts Log.color(:magenta, "Example ") << Log.color(:green, example) + " -- " + Log.color(:blue, example_dir[task_name][example])
-
-            inputs = self.example(task_name, example)
-
-            inputs.each do |input, type, file|
-                case type
-                when :tsv, :array, :text
-                  lines = file.read.split("\n")
-                  head = lines[0..5].compact * "\n\n"
-                  head = head[0..500]
-                  puts Misc.format_definition_list_item(input, head, 1000, -1, :blue).gsub(/\n\s*\n/,"\n") 
-                  puts '...' if lines.length > 6
-                else
-                  puts Misc.format_definition_list_item(input, file.read, 80, 20, :blue)
-                end
+        self.examples[task_name].each do |example|
+
+          puts Log.color(:magenta, "Example ") << Log.color(:green, example) + " -- " + Log.color(:blue, example_dir[task_name][example])
+
+          inputs = self.example(task_name, example)
+
+          inputs.each do |input, type, file|
+            case type
+            when :tsv, :array, :text
+              lines = file.read.split("\n")
+              head = lines[0..5].compact * "\n\n"
+              head = head[0..500]
+              puts Misc.format_definition_list_item(input, head, 1000, -1, :blue).gsub(/\n\s*\n/,"\n") 
+              puts '...' if lines.length > 6
+            else
+              puts Misc.format_definition_list_item(input, file.read, Log.terminal_width, 20, :blue)
             end
-            puts
           end
+          puts
         end
       end
+    end
   end
 
   def SOPT_str(task)

data/share/rbbt_commands/workflow/cmd

@@ -51,8 +51,8 @@ end
 
 prev = []
 
-$previous_commands << 'cmd'
-$previous_commands << workflow
+$previous_commands << ' cmd'
+$previous_commands << ' '  << workflow
 
 begin
   while ARGV.any?

data/share/rbbt_commands/workflow/forget_deps

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+require 'rbbt/workflow'
+
+$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Make a job forget all its dependencies and archive their meta-data
+
+$ #{$0} [options] <job_path>
+
+-h--help Print this help
+
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+path = ARGV[0]
+
+raise ParameterException, "No path given" if path.nil?
+step = Workflow.load_step path
+
+
+iif step.rec_dependencies
+
+step.archive_deps
+step.set_info :dependencies, []

data/share/rbbt_commands/workflow/task

@@ -15,7 +15,7 @@ def report_options(options)
   end
 end
 
-def usage(workflow = nil, task = nil, exception=nil)
+def usage(workflow = nil, task = nil, exception=nil, abridge = false)
   puts SOPT.doc
   puts
   if workflow.nil?
@@ -25,7 +25,7 @@ def usage(workflow = nil, task = nil, exception=nil)
 
   if task.nil?
     workflow.load_tasks if workflow.respond_to? :load_tasks
-    workflow.doc
+    workflow.doc nil, abridge
     puts
     puts "E.g. rbbt workflow task #{workflow.to_s} #{workflow.tasks.keys.first.to_s} -h"
   else
@@ -40,7 +40,7 @@ def usage(workflow = nil, task = nil, exception=nil)
       puts Misc.format_paragraph workflow.documentation[:description] 
     end
     puts
-    workflow.doc(task)
+    workflow.doc(task, abridge)
   end
 
   print_error(exception.message, exception.backtrace) if exception
@@ -171,6 +171,7 @@ can redo it using the `clean` parameter, this cleans the last step of the task.
 The `recursive_clean` cleans all the job dependency steps recursively.
 
 -h--help Show this help
+-ha--abridge Abridge help
 -wd--workdir* Change the working directory of the workflow
 -wda--workdir_all* Change the working directory of ALL workflow
 -as--array_separator* Change the character that separates elements of Arrays, ',', '|', or '\\n' by default
@@ -265,14 +266,14 @@ namespace = nil, nil
 
 case 
 when task.nil?
-  usage workflow and exit 0
+  usage workflow, nil, nil, options[:abridge] and exit 0
 else
   task_name = task.to_sym
   begin
     task = workflow.tasks[task_name]
     raise Workflow::TaskNotFoundException.new workflow, task_name if task.nil?
   rescue Workflow::TaskNotFoundException
-    usage workflow 
+    usage workflow, nil, nil, options[:abridge]
 
     puts
     puts Log.color :magenta, "## Error"
@@ -284,7 +285,7 @@ else
   end
 end
 
-usage workflow, task and exit 0 if help
+usage workflow, task, nil, options[:abridge] and exit 0 if help
 
 name = options.delete(:jobname) 
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.26.139
+  version: 5.26.140
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-07 00:00:00.000000000 Z
+date: 2020-02-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -322,6 +322,7 @@ files:
 - lib/rbbt/workflow/definition.rb
 - lib/rbbt/workflow/doc.rb
 - lib/rbbt/workflow/examples.rb
+- lib/rbbt/workflow/integration/cromwell.rb
 - lib/rbbt/workflow/integration/nextflow.rb
 - lib/rbbt/workflow/remote/client.rb
 - lib/rbbt/workflow/remote/remote_step.rb
@@ -414,6 +415,7 @@ files:
 - share/rbbt_commands/workflow/archive_all
 - share/rbbt_commands/workflow/cmd
 - share/rbbt_commands/workflow/example
+- share/rbbt_commands/workflow/forget_deps
 - share/rbbt_commands/workflow/info
 - share/rbbt_commands/workflow/init
 - share/rbbt_commands/workflow/install