rbbt-util 5.21.127 → 5.21.128

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2b39b4950994d49ab0541d19a8e7d3ac45245c1e
-  data.tar.gz: 28ac4746e480b19a5c9005671f15a29a97586f46
+  metadata.gz: d2e38a07f3d52f96131185d87edc11c40f7a5ea5
+  data.tar.gz: 920dab12666bcce73c7121959c22d99a0c2b2b57
 SHA512:
-  metadata.gz: 1722d689b494dedf1095a53f288b063dd12c2aa1ce01a05c3e92b17172266facd7445879d10e75d7aac65ebe22c502d37398e9345e19ffaf7297d680400b84df
-  data.tar.gz: 57e25447236ba7c6e5f28be95c687513c1259f742ddbdbe0d7957b695b9c17177e3c0105c821638871afec07e2249a3a9b2ff6c5015c9288ff0c018664fc4765
+  metadata.gz: 98919775334053530f302b4e98d31cefcec11bc38f3b1809c27ffec15befccd2a6819841aaa15389334fb2813fdf95253037696c0488de33f7d3b50c5ec98715
+  data.tar.gz: 1e2eae9c61741de2c300b935efd98009ba9babb8e5947d7b4c048a4be1eabfc28c2f2e79021f1b77df442340a7ed7cbbf3bf414501408141b00129c5e41df556

data/lib/rbbt/workflow.rb

@@ -355,9 +355,11 @@ module Workflow
 
   def set_step_dependencies(step)
     if step.info.include? :dependencies
-      step.dependencies = step.info[:dependencies].collect do |task, job, path|
-        next if job.nil?
-        load_step(path) 
+      Misc.insist do
+        step.dependencies = step.info[:dependencies].collect do |task, job, path|
+          next if job.nil?
+          load_step(path) 
+        end
       end
     end
   end

data/lib/rbbt/workflow/accessor.rb

@@ -367,12 +367,12 @@ class Step
   end
 
   def dirty_files
+    canfail_paths = self.canfail_paths
     dirty_files = rec_dependencies.reject{|dep|
       (defined?(WorkflowRESTClient) && WorkflowRESTClient::RemoteStep === dep) || 
         (dep.path && (Open.exists?(dep.path) || Open.remote?(dep.path))) || 
-        (dep.error? && ! dep.recoverable_error?)
+        ((dep.error? || dep.aborted?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
     }
-
   end
 
   def dirty?
@@ -550,6 +550,7 @@ module Workflow
     inputs = rec_inputs(name).uniq
     input_types = rec_input_types(name)
     input_descriptions = rec_input_descriptions(name)
+    input_use = rec_input_use(name)
     input_defaults = rec_input_defaults(name)
     input_options = rec_input_options(name)
     export = case
@@ -575,6 +576,7 @@ module Workflow
       :input_descriptions => input_descriptions,
       :input_defaults => input_defaults,
       :input_options => input_options,
+      :input_use => input_use,
       :result_type => result_type,
       :result_description => result_description,
       :dependencies => dependencies
@@ -696,6 +698,35 @@ module Workflow
       acc
     }.tap{|h| IndiferentHash.setup(h)}
   end
+  
+  def rec_input_use(taskname)
+    task = task_from_dep(taskname)
+    deps = rec_dependencies(taskname)
+    inputs = {}
+    task.inputs.each do |input|
+      name = task.name
+      workflow = (task.workflow || self).to_s
+
+      inputs[input] ||= {}
+      inputs[input][workflow] ||= []
+      inputs[input][workflow]  << name
+    end
+
+    dep_inputs = Task.dep_inputs deps, self
+
+    dep_inputs.each do |dep,is|
+      name = dep.name
+      workflow = dep.workflow
+
+      is.each do |input|
+        inputs[input] ||= {}
+        inputs[input][workflow] ||= []
+        inputs[input][workflow]  << name
+      end
+    end
+
+    inputs
+  end
 
   def rec_input_descriptions(taskname)
     rec_inputs = rec_inputs(taskname)

data/lib/rbbt/workflow/step/dependencies.rb

@@ -126,11 +126,12 @@ class Step
       end
 
       if dependency.aborted? or (dependency.error? and dependency.recoverable_error?) or (!Open.remote?(dependency.path) && dependency.missing?)
+        Log.warn "Cleaning dep. #{Log.color :red, dependency.task_name.to_s}"
         dependency.clean
         raise TryAgain
       end
 
-      if not dependency.started?
+      if ! dependency.started? && ! dependency.error?
         log_dependency_exec(dependency, :starting)
         dependency.run(true)
         raise TryAgain
@@ -157,6 +158,7 @@ class Step
       end
 
     rescue TryAgain
+      Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{dependency.messages.last}"
       retry
     rescue Aborted, Interrupt
       Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
@@ -267,6 +269,24 @@ class Step
     end
   end
 
+  def canfail_paths
+    if info[:canfail_paths]
+      Set.new(info[:canfail_paths])
+    else
+      canfail_paths = Set.new
+      all_deps = rec_dependencies + [self]
+      all_deps.each do |dep|
+        next if canfail_paths.include? dep.path
+        next unless ComputeDependency === dep && dep.canfail?
+        canfail_paths << dep.path
+        canfail_paths += dep.rec_dependencies.collect{|d| d.path }
+      end
+      canfail_paths
+      set_info :canfail_paths, canfail_paths.to_a
+      canfail_paths
+    end
+  end
+
   def run_dependencies
     dep_step = {}
 
@@ -281,11 +301,17 @@ class Step
       dep.rec_dependencies
     end.compact.flatten.uniq
 
+    canfail_paths = self.canfail_paths
+
     seen_paths = Set.new
     all_deps.uniq.each do |step|
       next if seen_paths.include? step.path
       seen_paths << step.path
-      Step.prepare_for_execution(step) unless step == self
+      begin
+        Step.prepare_for_execution(step) unless step == self
+      rescue DependencyError
+        raise $! unless canfail_paths.include? step.path
+      end
       next unless step.dependencies and step.dependencies.any?
       step.dependencies.each do |step_dep|
         next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and step_dep.compute == :nodup)
@@ -346,7 +372,11 @@ class Step
     Log.medium "Processing pre dependencies: #{Misc.fingerprint(pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
     pre_deps.each do |step|
       next if compute_deps.include? step
-      execute_and_dup(step, dep_step, false)
+      begin
+        execute_and_dup(step, dep_step, false)
+      rescue Exception
+        raise $! unless canfail_paths.include?(step.path)
+      end
     end
 
     Log.medium "Computing pre dependencies: #{Misc.fingerprint(compute_pre_deps)} - #{Log.color :blue, self.path}" if compute_pre_deps.any?
@@ -357,7 +387,11 @@ class Step
     Log.medium "Processing last dependencies: #{Misc.fingerprint(last_deps)} - #{Log.color :blue, self.path}" if last_deps.any?
     last_deps.each do |step|
       next if compute_deps.include? step
-      execute_and_dup(step, dep_step)
+      begin Exception
+        execute_and_dup(step, dep_step) 
+      rescue 
+        raise $! unless canfail_paths.include? step.path
+      end
     end
 
     Log.medium "Computing last dependencies: #{Misc.fingerprint(compute_last_deps)} - #{Log.color :blue, self.path}" if compute_last_deps.any?
@@ -365,6 +399,13 @@ class Step
       run_compute_dependencies(type, list, dep_step)
     end
 
+    Log.medium "Aborting waiting dangling dependencies"
+    all_deps.each do |dep|
+      next if dep.done?
+      next unless canfail_paths.include? dep.path
+      dep.abort if dep.waiting?
+    end
+
   end
 
   def stop_dependencies

data/lib/rbbt/workflow/step/run.rb

@@ -24,6 +24,14 @@ class Step
     new_inputs = inputs.collect do |i| 
       begin
         if Step === i
+          if i.error?
+            e = i.get_exception
+            if e
+              raise e
+            else
+              raise DependencyError, "Error in dep. #{Log.blue e.path}"
+            end
+          end
           step = true
           i.produce unless i.done? || i.error? || i.started?
           if i.done?
@@ -106,11 +114,12 @@ class Step
 
     outdated_time  = []
     outdated_dep  = []
+    canfail_paths = self.canfail_paths
     checks.each do |dep| 
       if dep.done? && self.done? && (File.mtime(dep.path) > File.mtime(self.path))
         outdated_time << dep
       end
-      if ! dep.done? || ! dep.updated?
+      if (! dep.done? && ! canfail_paths.include?(dep.path)) || ! dep.updated?
         outdated_dep << dep
       end
     end

data/share/Rlib/util.R

@@ -523,7 +523,44 @@ rbbt.png_plot <- function(filename, p, width=500, height=500, ...){
     eval(parse(text=p));
 }
 
-rbbt.heatmap <- function(filename, data, width=500, height=500, take_log=FALSE, stdize=TRUE, ...){
+rbbt.pheatmap <- function(filename, data, width=800, height=800, take_log=FALSE, stdize=FALSE, positive=FALSE, ...){
+    rbbt.require('pheatmap')
+
+    opar = par()
+    png(filename=filename, width=width, height=height);
+
+    data = as.matrix(data)
+    data[is.nan(data)] = NA
+
+    data = data[rowSums(is.na(data))==0, ]
+
+    if (take_log){
+        for (study in colnames(data)){
+            skip = sum(data[, study] <= 0) != 0
+            if (!skip){
+                data[, study] = log(data[, study])
+            }
+        }
+        data = data[, colSums(is.na(data))==0]
+    }
+
+
+    if (stdize){
+        rbbt.require('pls')
+        data = stdize(data)
+    }
+
+    if (positive){
+        pheatmap(data,color= colorRampPalette(c("white", "red"))(100), ...)
+    }else{
+        pheatmap(data, ...)
+    }
+
+    dev.off();
+    par(opar)
+}
+
+rbbt.heatmap <- function(filename, data, width=800, height=800, take_log=FALSE, stdize=FALSE, ...){
     opar = par()
     png(filename=filename, width=width, height=height);
 

data/share/rbbt_commands/stat/density

@@ -24,15 +24,13 @@ end
 
 values = file.read.split("\n").collect{|v| v.to_f}
 
-res = TmpFile.with_file do |tmp|
+res = TmpFile.with_file nil, :extension => 'png' do |tmp|
   R.run <<-EOF
 values = #{R.ruby2R values}
 d = density(values)
-df = data.frame(x=d$x, y=d$y)
-rbbt.tsv.write(file='#{ tmp }', df)
+rbbt.png_plot(filename = '#{tmp}', 'plot(d)')
   EOF
 
-  Open.read(tmp)
+  `op '#{tmp}'` 
 end
 
-puts res.split("\n")[1..-1].collect{|l| l.split("\t")[1,2] * "\t"} * "\n"

data/share/rbbt_commands/stat/heatmap

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+Produce a heatmap
+
+$ rbbt stat heatmap [options] <filename.tsv> [file.png]
+
+-h--help Print this help
+-w--width* Image Width
+-h--height* Image Height
+-s--size* Image Size (Height and Width)
+-l--logs Take logs
+-p--positive single color heatmap (positive values
+
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.usage
+  end
+  exit 0
+end
+
+file, png = ARGV
+
+raise "No file" if file.nil?
+
+width, height, size, logs, positive = options.values_at :width, :height, :size, :logs, :positive
+
+size = 800 if size.nil?
+width = size if width.nil?
+height = size if height.nil?
+logs = false if logs.nil?
+positive = false if positive.nil?
+
+
+require 'rbbt/util/R'
+if png
+    R.run <<-EOF
+data = rbbt.tsv('#{file}')
+rbbt.pheatmap('#{png}', data, positive=#{R.ruby2R positive})
+    EOF
+else
+    TmpFile.with_file(nil, :extension => 'png') do |png|
+        R.run <<-EOF
+data = rbbt.tsv('#{file}')
+rbbt.pheatmap('#{png}', data, width=#{ width }, height=#{ height }, take_log=#{R.ruby2R logs}, positive=#{R.ruby2R positive})
+        EOF
+        `op #{png}`
+    end
+end

data/share/rbbt_commands/workflow/cmd

@@ -9,8 +9,18 @@ workflow = ARGV.shift
 wf = Workflow.require_workflow workflow
 dir = $command_dir = wf.libdir.share.rbbt_commands
 
+def prev_dir(prev)
+    rbbt_command_dir = $rbbt_command_dir
+
+    prev.each do |previous_command|
+        rbbt_command_dir = rbbt_command_dir[previous_command]
+    end
+
+    rbbt_command_dir
+end
+
 def commands(prev)
-    rbbt_command_dir = $command_dir
+    rbbt_command_dir = prev_dir(prev)
 
     command_file_dirs = rbbt_command_dir.find_all
     command_files = command_file_dirs.collect{|d| d.glob('*') }.flatten
@@ -24,6 +34,9 @@ begin
   while ARGV.any?
     command = ARGV.shift
     case
+    when File.directory?(dir[command].find)
+      prev << command
+      dir = dir[command]
     when File.directory?(dir[command].find)
       prev << command
       dir = dir[command]
@@ -54,7 +67,13 @@ puts "  #{File.basename($0)} #{$previous_commands * " "} cmd"
 puts 
 puts Log.color :magenta, "Subcommands:"
 puts 
+
 commands(prev).each do |command|
+  directory = File.directory? dir[command].find
+  if directory
+    puts "  " << Log.color(:blue, command)
+  else
     puts "  " << command
+  end
 end
     

data/test/rbbt/test_workflow.rb

@@ -222,5 +222,10 @@ class TestWorkflow < Test::Unit::TestCase
     end
   end
 
+  def test_rec_input_use
+    assert TestWF.rec_input_use(:double_dep).include?(:times)
+    assert TestWF.rec_input_use(:double_dep)[:times].include?(TestWF)
+    assert TestWF.rec_input_use(:double_dep)[:times][TestWF].include?(:repeat)
+  end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.127
+  version: 5.21.128
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-08 00:00:00.000000000 Z
+date: 2018-05-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -358,6 +358,7 @@ files:
 - share/rbbt_commands/rsync
 - share/rbbt_commands/stat/abs
 - share/rbbt_commands/stat/density
+- share/rbbt_commands/stat/heatmap
 - share/rbbt_commands/stat/log
 - share/rbbt_commands/stat/pvalue.qqplot
 - share/rbbt_commands/study/maf2study