rbbt-util 5.21.98 → 5.21.99

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12d10e9ae9d12ed354fe3d8df24a14ea030bb1eb
-  data.tar.gz: 13416b83ec00f7dae0fc97479746cd636cf53a5c
+  metadata.gz: 7dd511eb01b14817584552dad9178ede2319c65b
+  data.tar.gz: 8931609250cca85686c0be1e35b3d9cdcba8aca6
 SHA512:
-  metadata.gz: d86681d7392cd8a0dae501a10a792e91ca93a026adb918d386d5a5b83916525f66947d233bbe5302cf65c2c02ae2b19582ce52872946b98265c5d8752f02ea2f
-  data.tar.gz: a93dc769c50f8d08e74d8088caa7c953aee25a78b47576f0015fd8b73f113d736a234534e1faf791eee6e938686db4dcd03a9f1ac44e6643ce5aaca8b8da0a56
+  metadata.gz: 298f9fffa765f4eb2c2cd92a85289c148ab8e49d81e4a509341fd9256222cd8d3ff42fd81217d040241dd2960f4ff743853cf5ec0ad8ae12094dbca7d11e0e2e
+  data.tar.gz: 06ef1631872f80edce0c3dc759d650c83466666a70ec123242d0bcce6c7ea225417b5bcc8561ae285934ec8bfd570bb980ccccb1483128f7a16ce58444a5b9ab

data/bin/rbbt

@@ -48,8 +48,12 @@ $ rbbt <command> <subcommand> ... -a --arg1 --arg2='value' --arg3 'another-value
 --locate_file           #{Log.color :yellow, "Report the location of the script instead of executing it"}
 --dump_mem*             #{Log.color :yellow, "Dump strings in memory each second into file"}
 -nolock--no_lock_id            #{Log.color :yellow, "Do not track lockfiles with ids (prevent stale file handlers for high-througput and high-concurrency)"}
+-ji--jobname_as_inputs  #{Log.color :yellow, "Use inputs as part of the jobname in workflows instead of digesting them"}
 EOF
 
+if options[:jobname_as_inputs]
+  ENV["RBBT_INPUT_JOBNAME"] = "true"
+end
 
 locate = options.delete :locate_file
 

data/lib/rbbt/resource/path.rb

@@ -1,4 +1,5 @@
 require 'rbbt/resource/util'
+require 'rbbt/util/misc/indiferent_hash'
 require 'yaml'
 
 module Path

data/lib/rbbt/util/R.rb

@@ -22,8 +22,7 @@ source('#{UTIL}');
     EOF
 
     require_sources  = source.collect{|source|
-      source = R::LIB_DIR["plot.R"] if source == :plot
-      source = R::LIB_DIR["svg.R"] if source == :svg
+      source = R::LIB_DIR["#{source.to_s}.R"] if R::LIB_DIR["#{source.to_s}.R"].exists?
       "source('#{source}')"
     } * ";\n" if Array === source and source.any?
 
@@ -150,7 +149,7 @@ module TSV
     source = [source] unless Array === source 
 
     require_sources  = source.collect{|source|
-      source = R::LIB_DIR["plot.R"] if source == :plot
+      source = R::LIB_DIR["#{source.to_s}.R"] if R::LIB_DIR["#{source.to_s}.R"].exists?
       "source('#{source}')"
     } * ";\n" if Array === source and source.any?
 

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -70,19 +70,18 @@ class RbbtProcessQueue
 
       status = nil
       begin
-        @current = Process.fork do
-          run
-        end
-        @asked = false
 
         initial = Misc.memory_use(Process.pid)
         memory_cap = multiplier * initial
         Log.debug "Worker for #{Process.pid} started with pid #{@current} -- initial: #{initial} - multiplier: #{multiplier} - cap: #{memory_cap}"
 
+        @asked = false
+        @monitored = false
         @monitor_thread = Thread.new do
           begin
             while true
-              current = Misc.memory_use(@current) 
+              @monitored = true
+              current = @current ? 0 : Misc.memory_use(@current) 
               if current > memory_cap and not @asked
                 Log.medium "Worker #{@current} for #{Process.pid} asked to respawn -- initial: #{initial} - multiplier: #{multiplier} - cap: #{memory_cap} - current: #{current}"
                 RbbtSemaphore.synchronize(@callback_queue.write_sem) do
@@ -97,6 +96,13 @@ class RbbtProcessQueue
           end
         end
 
+        while ! @monitored
+          sleep 0.1
+        end
+        @current = Process.fork do
+          run
+        end
+
         while true
           pid, status = Process.waitpid2 @current
           code = status.to_i >> 8 
@@ -114,7 +120,7 @@ class RbbtProcessQueue
         Log.exception $!
         raise $!
       ensure
-        @monitor_thread.kill
+        @monitor_thread.kill if @monitor_thread
         Process.kill "INT", @current if Misc.pid_exists? @current
         @callback_queue.close_write if @callback_queue 
       end

data/lib/rbbt/workflow.rb

@@ -337,7 +337,7 @@ module Workflow
       real_inputs[k] = v 
     end
 
-    if real_inputs.empty?
+    if real_inputs.empty? and not Workflow::TAG == :inputs
       step_path = step_path taskname, jobname, [], [], task.extension
       input_values = task.take_input_values(inputs)
     else

data/lib/rbbt/workflow/accessor.rb

@@ -351,6 +351,10 @@ class Step
     Open.exists?(path) or Open.exists?(pid_file) #or Open.exists?(info_file)
   end
 
+  def waiting?
+    Open.exists?(info_file) and not started?
+  end
+
   def dirty?
     status = self.status
     return true if done? and not status == :done and not status == :noinfo
@@ -359,7 +363,7 @@ class Step
     if dirty_files.any?
       true
     else
-      false
+      ! self.updated?
     end
   end
 
@@ -759,13 +763,10 @@ module Workflow
                  when Proc
                    if DependencyBlock === dependency
                      orig_dep = dependency.dependency 
-                     if Hash === orig_dep.last
-                       options = orig_dep.last
-                       compute = options[:compute]
-                     else
-                       options = {}
-                       compute = nil
-                     end
+                     wf, task_name, options = orig_dep
+
+                     options = {} if options.nil?
+                     compute = options[:compute]
 
                      options = IndiferentHash.setup(options.dup)
                      dep = dependency.call jobname, options.merge(_inputs), real_dependencies
@@ -775,7 +776,8 @@ module Workflow
                      new_=[]
                      dep.each{|d| 
                        if Hash === d
-                         d[:workflow] ||= self 
+                         d[:workflow] ||= wf 
+                         d[:task] = task_name
                          inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, d[:workflow].task_info(d[:task])) 
                          d = d[:workflow].job(d[:task], d[:jobname], inputs) 
                        end
@@ -802,7 +804,7 @@ module Workflow
     real_dependencies.flatten.compact
   end
 
-  TAG = :hash
+  TAG = ENV["RBBT_INPUT_JOBNAME"] == "true" ? :inputs : :hash
   def step_path(taskname, jobname, inputs, dependencies, extension = nil)
     raise "Jobname makes an invalid path: #{ jobname }" if jobname =~ /\.\./
     if inputs.length > 0 or dependencies.any?
@@ -810,6 +812,19 @@ module Workflow
                        when :hash
                          hash_str = Misc.obj2digest({:inputs => inputs, :dependencies => dependencies})
                          jobname + '_' << hash_str
+                       when :inputs
+                         all_inputs = {}
+                         inputs.zip(self.task_info(taskname)[:inputs]) do |i,f|
+                           all_inputs[f] = i
+                         end
+                         dependencies.each do |dep|
+                           ri = dep.recursive_inputs
+                           ri.zip(ri.fields).each do |i,f|
+                             all_inputs[f] = i
+                           end
+                         end
+
+                         all_inputs.any? ? jobname + '_' << Misc.obj2str(all_inputs) : jobname
                        else
                          jobname
                        end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -80,13 +80,14 @@ class Step
       return if status == 'streaming' and job.running?
     end
 
-    if ((status == 'error' || job.aborted?) && job.recoverable_error?) || job.dirty?
+    if ((status == 'error' || job.aborted?) && job.recoverable_error?) || job.dirty? 
       job.clean 
     end
 
     (job.init_info and job.dup_inputs) unless status == 'done' or job.started?
 
-    raise DependencyError, job if job.error?
+    canfail = ComputeDependency === job and Array === job.compute and job.compute.include? :canfail
+    raise DependencyError, job if job.error? and not canfail
   end
 
   def log_dependency_exec(dependency, action)
@@ -119,6 +120,12 @@ class Step
         return
       end
 
+      if dependency.aborted?
+        log_dependency_exec(dependency, "aborted (clean)")
+        dependency.clean
+        raise TryAgain
+      end
+
       if not dependency.started?
         log_dependency_exec(dependency, :starting)
         dependency.run(true)
@@ -127,12 +134,6 @@ class Step
 
       dependency.grace
 
-      if dependency.aborted?
-        log_dependency_exec(dependency, "aborted (clean)")
-        dependency.clean
-        raise TryAgain
-      end
-
       if dependency.error?
         log_dependency_exec(dependency, :error)
         raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
@@ -161,7 +162,6 @@ class Step
       raise $!
     rescue Exception
       Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s } -- #{$!.message}"
-      #Log.exception $!
       raise $!
     end
   end
@@ -189,12 +189,16 @@ class Step
       type, *rest = type
     end
 
+    canfail = rest && rest.include?(:canfail)
+
     case type
     when :produce, :no_dup
       list.each do |step|
         Misc.insist do
           begin
             step.produce
+          rescue RbbtException
+            raise $! unless canfail
           rescue Exception
             step.exception $!
             if step.recoverable_error?
@@ -211,11 +215,22 @@ class Step
       cpus = 5 if cpus.nil?
       cpus = list.length / 2 if cpus > list.length / 2
 
-      Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path}", :respawn => :always) do |dep|
+      respawn = rest && rest.include?(:respawn)
+      respawn = false if rest && rest.include?(:norespawn)
+      respawn = rest && rest.include?(:always_respawn)
+      respawn = :always if respawn.nil?
+
+      Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path}", :respawn => respawn) do |dep|
         Misc.insist do
           begin
             dep.produce 
             Log.warn "Error in bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
+          rescue Exception
+            if canfail
+              Log.warn "Allowing failing of #{dep.path}: #{dep.messages.last}"
+            else
+              raise $!
+            end
           rescue Aborted
             dep.abort
             Log.warn "Aborted bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?

data/lib/rbbt/workflow/step/run.rb

@@ -86,7 +86,15 @@ class Step
   end
 
   def checks
-    rec_dependencies.collect{|dependency| (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency) ? nil : dependency.path }.compact.uniq
+    #rec_dependencies.collect{|dependency| (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency) ? nil : dependency.path }.compact.uniq
+    rec_dependencies.
+      select{|dependency| ! (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency) }.
+      select{|dependency| ! dependency.error? }.
+      collect{|dependency| dependency.path }.uniq
+  end
+
+  def updated?
+    done? and checks.select{|path| File.mtime(path) > File.mtime(self.path)  }.empty?
   end
 
   def kill_children
@@ -114,13 +122,16 @@ class Step
     begin
       @mutex.synchronize do
         no_load = :stream if no_load
+
+        Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
         result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do 
           if Step === Step.log_relay_step and not self == Step.log_relay_step
             relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
           end
 
+          Open.write(pid_file, Process.pid.to_s) unless Open.exists? pid_file
+
           @exec = false
-          Open.write(pid_file, Process.pid.to_s)
           init_info
 
           log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task.name.to_s || ""}"

data/share/Rlib/util.R

@@ -33,7 +33,7 @@ rbbt.ruby <- function(code, load = TRUE, flat = FALSE, type = 'tsv', ...){
   }
 }
 
-rbbt.job <- function(workflow, task, load=TRUE, flat = FALSE, type = 'tsv', jobname="R.Default", code='', ...){
+rbbt.job <- function(workflow, task, load=TRUE, flat = FALSE, type = 'tsv', jobname="Default", code='', ...){
 
     str = "require 'rbbt/workflow'"
 
@@ -43,26 +43,47 @@ rbbt.job <- function(workflow, task, load=TRUE, flat = FALSE, type = 'tsv', jobn
 
     args_list = list(...)
     args_strs = c()
+    tmp_files = c()
+
     for (input in names(args_list)){
         value = args_list[[input]]
         input = sub('input\\.', '', input)
-        if (!is.numeric(value)){
-            if (value == TRUE){
-                value = 'true'
-            }else{
-                if (value == FALSE){
-                    value = 'false'
+        if (is.vector(value) && length(value) > 1){
+            file = tempfile()
+            writeLines(value, file)
+            tmp_files = c(tmp_files, file)
+            value = paste("Open.read('", file, "').split(\"\\n\")", sep="")
+        }else{
+            if (!is.numeric(value)){
+                if (all(value %in% TRUE)){
+                    value = 'true'
                 }else{
-                    value = paste("'", value, "'", sep="")
+                    if (all(value %in% FALSE)){
+                        value = 'false'
+                    }else{
+                        if (is.data.frame(value)){
+                            file = tempfile()
+                            rbbt.tsv.write(file, value)
+                            tmp_files = c(tmp_files, file)
+                            value = paste("TSV.open('", file, "')", sep="")
+                        }else{
+                            value = paste("'", value, "'", sep="")
+                        }
+                    }
                 }
             }
         }
         args_strs = c(args_strs, paste(":",input,' => ',value, sep=""))
     }
 
-    args_str = paste(args_strs, sep=",")
+    args_str = paste(args_strs, collapse=",")
     str = paste(str, paste('wf.job(:', task, ", '", jobname, "', ", args_str,').produce.path', sep=""), sep="\n")
-    return(rbbt.ruby(str, load, flat, type));
+
+    res = rbbt.ruby(str, load, flat, type)
+
+    unlink(tmp_files)
+
+    return(res);
 }
 
 rbbt.ruby.substitutions <- function(script, substitutions = list(), ...){
@@ -134,11 +155,23 @@ rbbt.tsv <- function(filename, sep = "\t", comment.char ="#", row.names=1, check
   columns = rbbt.tsv.columns(filename, sep, comment.char=comment.char)
   if (! is.null(columns)){
       names(data) <- columns[2:length(columns)];
+      attributes(data)$key.field = substring(columns[1],2);
   }
 
   return(data);
 }
 
+rbbt.tsv.comma <- function(tsv){
+    for (c in names(tsv)){
+        v = tsv[,c]
+        if (is.character(v)){
+            v = gsub('\\|', ', ', v)
+            tsv[,c] = v
+        }
+    }
+    return(tsv)
+}
+
 rbbt.tsv.numeric <- function(filename, sep="\t", ...){
 
     columns = rbbt.tsv.columns(filename, sep)
@@ -156,6 +189,8 @@ rbbt.tsv2matrix <- function(data){
 }
 
 rbbt.tsv.write <- function(filename, data, key.field = NULL, extra_headers = NULL){
+
+  if (is.null(key.field)){ key.field = attributes(data)$key.field;}
   if (is.null(key.field)){ key.field = "ID";}
 
   f = file(filename, 'w');

data/share/rbbt_commands/tsv/read_excel

@@ -20,6 +20,7 @@ Use - to read from STDIN
 -z--zipped Assume data is zipped when merging (default is true)
 -s2--sep2* Second level separator. Default /[,|]\\s?/
 -h--help Print this help
+-s--sheet* Sheet to extract
 
 EOF
 if options[:help]

data/share/rbbt_commands/workflow/prov

@@ -52,7 +52,7 @@ def report_msg(status, name, path)
   task = Log.color(:yellow, parts.pop)
   workflow = Log.color(:magenta, parts.pop)
 
-  if not Open.remote?(path) and $main_mtime and ($main_mtime - File.mtime(path)) < 0
+  if not Open.remote?(path) and (File.exists?(path) and $main_mtime and ($main_mtime - File.mtime(path)) < 0)
     status_msg(status) << " " << [workflow, task, path] * " " << " (#{Log.color(:red, "Mtime out of sync") })\n"
   else
     status_msg(status) << " " << [workflow, task, path] * " " << "\n"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.98
+  version: 5.21.99
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-06 00:00:00.000000000 Z
+date: 2017-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake