rbbt-util 5.38.1 → 5.40.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1121593dec25c29d7986047727d555ed0e75df6b158f9f1332764ad4886462c5
-  data.tar.gz: c8e39ef65e23445cf5d0c8817af961a9206016c5e3665d87e042f0fadec2b8df
+  metadata.gz: '088ac190b36a176e420d4bb1d7b8d28d777133568e45530219f6a0212a1a1144'
+  data.tar.gz: 47bdc31d247e5fd5d944f493690e8181bcc236cee74c61fefd5ce25711edf7da
 SHA512:
-  metadata.gz: 22c87766544a52c9c6f0d6da369422a452f553d1a61b82a54674f676a3867a6c1d29028c2efd031563f982cdadbce56d36950c8e100bd34ac2a7e95b39af8a58
-  data.tar.gz: 0a79edc89dad388bca9b94dfeccba1df64c4294e84c4f19743608038e55f8800181911d787980adae3ec1c715a993430ccbf2bc07d6212485b89843ada93de74
+  metadata.gz: 9afce115c221695ee896c1e1f884be9a1ba0f0ae98789a485a906e438252afe8292c5422f3eef57d77089bf55cc0b425988ab9b479b1f422eaa0abb7005cfdfa
+  data.tar.gz: 4cf4d1e86ce3ab256609aff4c66a3412ea1a9c0c87db08fd5cdad1b98c95147d452cc69d96009c9791cf698ea43903873ed0e4cebbc861bda85097864bed68e8

data/lib/rbbt/hpc/batch.rb

@@ -1,5 +1,5 @@
 module HPC
-  class SBATCH < Exception; 
+  class BATCH_DRY_RUN < Exception; 
     attr_accessor :directory
     def initialize(directory)
       @directory = directory
@@ -12,24 +12,32 @@ module HPC
       HPC::SLURM
     when 'lsf'
       HPC::LSF
+    when 'pbs'
+      HPC::PBS
     when 'auto'
       case $previous_commands.last
       when 'slurm'
         HPC::SLURM
       when 'lsf'
         HPC::LSF
+      when 'pbs'
+        HPC::PBS
       else
         case Rbbt::Config.get(:batch_system, :batch, :batch_system, :hpc, :HPC, :BATCH).to_s.downcase
         when 'slurm'
           HPC::SLURM
         when 'lsf'
           HPC::LSF
+        when 'pbd'
+          HPC::PBS
         else
           case ENV["BATCH_SYSTEM"].to_s.downcase
           when 'slurm'
             HPC::SLURM
           when 'lsf'
             HPC::LSF
+          when 'pbs'
+            HPC::PBS
           end
         end
       end
@@ -564,8 +572,8 @@ env > #{batch_options[:fenv]}
     def run_job(job, options = {})
       system = self.to_s.split("::").last
 
-      batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run = Misc.process_options options, 
-        :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run,
+      batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run, orchestration_rules_file = Misc.process_options options, 
+        :batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run, :orchestration_rules,
         :batch_base_dir => File.expand_path(File.join('~/rbbt-batch')) 
 
       if (batch_job = job.info[:batch_job]) && job_queued(batch_job)
@@ -586,6 +594,8 @@ env > #{batch_options[:fenv]}
       workflow = job.original_workflow ||job.workflow
       task_name = job.original_task_name || job.task_name
 
+      options = options.merge(HPC::Orchestration.job_rules(HPC::Orchestration.orchestration_rules(orchestration_rules_file), job)) if orchestration_rules_file
+
       workflows_to_load = job.rec_dependencies.select{|d| Step === d}.collect{|d| d.workflow }.compact.collect(&:to_s) - [workflow.to_s]
 
       TmpFile.with_file(nil, remove_batch_dir, :tmpdir => batch_base_dir, :prefix => "#{system}_rbbt_job-#{workflow.to_s}-#{task_name}-") do |batch_dir|

data/lib/rbbt/hpc/lsf.rb

@@ -99,7 +99,7 @@ export BATCH_SYSTEM=#{batch_system}
         elsif dry_run
           STDERR.puts Log.color(:magenta, "To execute run: ") + Log.color(:blue, cmd)
           STDERR.puts Log.color(:magenta, "To monitor progress run (needs local rbbt): ") + Log.color(:blue, "rbbt lsf tail '#{batch_dir}'")
-          raise HPC::SBATCH, batch_dir
+          raise HPC::BATCH_DRY_RUN, batch_dir
         else
           Open.rm fsync
           Open.rm fexit

data/lib/rbbt/hpc/orchestrate/rules.rb

@@ -65,6 +65,13 @@ module HPC
       merge_rules(rules[workflow][task], workflow_rules)
     end
 
-
+    def self.job_rules(rules, job)
+      return {} if job.done? or job.error?
+      job_rules = task_specific_rules(rules, job.workflow.to_s, job.task_name.to_s)
+      job.dependencies.each do |dep|
+        job_rules = accumulate_rules(job_rules, job_rules(rules, dep))
+      end
+      job_rules
+    end
   end
 end

data/lib/rbbt/hpc/orchestrate.rb

@@ -22,6 +22,25 @@ module HPC
 
     end
 
+    def self.orchestration_rules(orchestration_rules_file = nil)
+      rules = {}
+      if orchestration_rules_file
+        if Open.exists?(orchestration_rules_file)
+          rules = Misc.load_yaml(orchestration_rules_file)
+        elsif Rbbt.etc.batch[orchestration_rules_file].exists?
+          rules = Misc.load_yaml(Rbbt.etc.batch[orchestration_rules_file])
+        elsif Rbbt.etc.batch[orchestration_rules_file + '.yaml'].exists?
+          rules = Misc.load_yaml(Rbbt.etc.batch[orchestration_rules_file + '.yaml'])
+        else
+          raise "Orchestration rules file not found: #{orchestration_rules_file}"
+        end
+      elsif Rbbt.etc.batch["default.yaml"].exists?
+        rules = Misc.load_yaml(Rbbt.etc.batch["default.yaml"])
+      end
+
+      IndiferentHash.setup(rules)
+    end
+
     def orchestrate_job(job, options)
       options.delete "recursive_clean"
       options.delete "clean_task"
@@ -33,26 +52,11 @@ module HPC
       options.delete "load_inputs"
       options.delete "provenance"
 
+
       Log.high "Prepare for exec"
       prepare_for_execution(job)
 
-      if orchestration_rules_file = options[:orchestration_rules]
-        if Open.exists?(orchestration_rules_file)
-          rules = Misc.load_yaml(orchestration_rules_file)
-        elsif Rbbt.etc.slurm(orchestration_rules_file).exists?
-          rules = Misc.load_yaml(Rbbt.etc.slurm(orchestration_rules_file))
-        elsif Rbbt.etc.slurm(orchestration_rules_file + '.yaml').exists?
-          rules = Misc.load_yaml(Rbbt.etc.slurm(orchestration_rules_file + '.yaml'))
-        else
-          raise "Orchestration rules file not found: #{options[:orchestration_rules]}"
-        end
-      elsif Rbbt.etc.slurm["default.yaml"].exists?
-        rules = Misc.load_yaml(Rbbt.etc.slurm["default.yaml"])
-      else
-        rules = {}
-      end
-
-      IndiferentHash.setup(rules)
+      rules = HPC::Orchestration.orchestration_rules(options[:orchestration_rules])
 
       batches = HPC::Orchestration.job_batches(rules, job)
       Log.high "Compute #{batches.length} batches"

data/lib/rbbt/hpc/pbs.rb

@@ -0,0 +1,177 @@
+require 'rbbt/hpc/batch'
+require 'rbbt/hpc/orchestrate'
+
+module HPC
+  module PBS 
+    extend HPC::TemplateGeneration
+    extend HPC::Orchestration
+
+    def self.batch_system
+      "PBS"
+    end
+
+    def self.batch_system_variables
+      <<-EOF
+let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
+let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") / ( (1024 * $TOTAL_PROCESORS) / $PBS_CPUS_PER_TASK )"
+MAX_MEMORY="$MAX_MEMORY_DEFAULT"
+[ ! -z $PBS_MEM_PER_CPU ] && let MAX_MEMORY="$PBS_MEM_PER_CPU * $PBS_CPUS_PER_TASK" 
+[ ! -z $PBS_MEM_PER_NODE ] && MAX_MEMORY="$PBS_MEM_PER_NODE"
+export MAX_MEMORY_DEFAULT
+export MAX_MEMORY
+export BATCH_JOB_ID=$PBS_JOBID
+export BATCH_SYSTEM=#{batch_system}
+
+cd ${PBS_O_WORKDIR}
+      EOF
+    end
+
+    def self.header(options = {})
+      options = options.dup
+
+      workdir    = Misc.process_options options, :workdir
+      batch_dir  = Misc.process_options options, :batch_dir
+      batch_name = Misc.process_options options, :batch_name
+
+      queue      = Misc.process_options options, :queue
+      account    = Misc.process_options options, :account
+      time       = Misc.process_options options, :time
+      nodes      = Misc.process_options options, :nodes
+
+      # PBS 
+      place      = Misc.process_options options, :place, :place => 'scatter'
+      system     = Misc.process_options options, :partition
+      filesystems = Misc.process_options options, :filesystems
+
+      filesystems = filesystems * "," if Array === filesystems
+
+      # NOT USED 
+      partition  = Misc.process_options options, :partition
+      task_cpus  = Misc.process_options options, :task_cpus
+      exclusive  = Misc.process_options options, :exclusive
+      highmem    = Misc.process_options options, :highmem
+      licenses   = Misc.process_options options, :licenses
+      constraint = Misc.process_options options, :constraint
+      gres       = Misc.process_options options, :gres
+
+      constraint     = [constraint, "highmem"].compact * "&" if highmem
+
+      mem            = Misc.process_options options, :mem
+      mem_per_cpu    = Misc.process_options options, :mem_per_cpu
+
+      fout       = File.join(batch_dir, 'std.out')
+      ferr       = File.join(batch_dir, 'std.err')
+
+      time = Misc.format_seconds Misc.timespan(time) unless time.include? ":"
+
+      qsub_params = {  "-l filesystems=" => filesystems,
+                       "-l system=" => system,
+                       "-l select=" => nodes,
+                       "-l place=" => place,
+                       "-l walltime=" => time,
+                       "-q " => queue,
+                       "-A " => account,
+                       "-o " => fout,
+                       "-e " => ferr,
+                       "-k doe" => true,
+                       # "cpus-per-task" => task_cpus,
+                       # "nodes" => nodes,
+                       # "time" => time,
+                       # "constraint" => constraint,
+                       # "exclusive" => exclusive,
+                       # "licenses" => licenses,
+                       # "gres" => gres,
+                       # "mem" => mem,
+                       # "mem-per-cpu" => mem_per_cpu,
+      }
+
+
+      header =<<-EOF
+#!/bin/bash
+      EOF
+
+      qsub_params.each do |name,value|
+        next if value.nil? || value == ""
+        if TrueClass === value
+          header << "#PBS #{name}" << "\n"
+        elsif Array === value
+          value.each do |v|
+            header << "#PBS #{name}\"#{v}\"" << "\n"
+          end
+        else
+          header << "#PBS #{name}\"#{value}\"" << "\n"
+        end
+      end
+
+      header
+    end
+
+    def self.run_template(batch_dir, dry_run)
+
+      fout   = File.join(batch_dir, 'std.out')
+      ferr   = File.join(batch_dir, 'std.err')
+      fjob   = File.join(batch_dir, 'job.id')
+      fdep   = File.join(batch_dir, 'dependencies.list')
+      fcfdep = File.join(batch_dir, 'canfail_dependencies.list')
+      fexit  = File.join(batch_dir, 'exit.status')
+      fsync  = File.join(batch_dir, 'sync.log')
+      fcmd   = File.join(batch_dir, 'command.batch')
+
+      return if Open.exists?(fexit)
+
+      Log.info "Issuing PBS file: #{fcmd}"
+      Log.debug Open.read(fcmd)
+
+      if File.exist?(fjob)
+        job = Open.read(fjob).to_i
+      else
+
+        dependencies = Open.read(fdep).split("\n") if File.exist? fdep
+        canfail_dependencies = Open.read(fcfdep).split("\n") if File.exist? fcfdep
+
+        normal_dep_str = dependencies && dependencies.any? ? "afterok:" + dependencies * ":" : nil
+        canfail_dep_str = canfail_dependencies && canfail_dependencies.any? ? "afterany:" + canfail_dependencies * ":" : nil
+
+        if normal_dep_str.nil? && canfail_dep_str.nil?
+          dep_str = ""
+        else
+          dep_str = '-W depend=' + [normal_dep_str, canfail_dep_str].compact * ","
+        end
+
+        cmd = "qsub #{dep_str} '#{fcmd}'"
+
+        if File.exist?(fout)
+          return
+        elsif dry_run
+          STDERR.puts Log.color(:magenta, "To execute run: ") + Log.color(:blue, "squb '#{fcmd}'")
+          STDERR.puts Log.color(:magenta, "To monitor progress run (needs local rbbt): ") + Log.color(:blue, "rbbt pbs tail '#{batch_dir}'")
+          raise HPC::BATCH_DRY_RUN, batch_dir
+        else
+          Open.rm fsync
+          Open.rm fexit
+          Open.rm fout
+          Open.rm ferr
+
+          job = CMD.cmd(cmd).read.scan(/\d+/).first.to_i
+          Log.debug "SBATCH job id: #{job}"
+          Open.write(fjob, job.to_s)
+          job
+        end
+      end
+    end
+
+    def self.job_status(job = nil)
+      if job.nil?
+        CMD.cmd("qstat").read
+      else
+        begin
+          CMD.cmd("qstat #{job}").read
+        rescue
+          ""
+        end
+      end
+    end
+
+  end
+end
+

data/lib/rbbt/hpc/slurm.rb

@@ -130,7 +130,7 @@ export BATCH_SYSTEM=#{batch_system}
         elsif dry_run
           STDERR.puts Log.color(:magenta, "To execute run: ") + Log.color(:blue, "sbatch '#{fcmd}'")
           STDERR.puts Log.color(:magenta, "To monitor progress run (needs local rbbt): ") + Log.color(:blue, "rbbt slurm tail '#{batch_dir}'")
-          raise HPC::SBATCH, batch_dir
+          raise HPC::BATCH_DRY_RUN, batch_dir
         else
           Open.rm fsync
           Open.rm fexit

data/lib/rbbt/hpc.rb

@@ -4,3 +4,4 @@ require 'rbbt/hpc/batch'
 require 'rbbt/hpc/orchestrate'
 require 'rbbt/hpc/slurm'
 require 'rbbt/hpc/lsf'
+require 'rbbt/hpc/pbs'

data/lib/rbbt/resource/util.rb

@@ -1,6 +1,6 @@
 module Path
 
-  def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
+  def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin', 'LICENSE'])
     #file = caller.reject{|l| 
     #  l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
     #  l =~ /rbbt\/resource\/path\.rb/ or

data/lib/rbbt/resource.rb

@@ -354,7 +354,7 @@ url='#{url}'
     path = File.expand_path(path)
     path += "/" if File.directory?(path)
     resource ||= Rbbt
-    locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys)
+    locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys).uniq
     locations -= [:current, "current"]
     locations << :current
     search_paths = IndiferentHash.setup(resource.search_paths)
@@ -374,7 +374,11 @@ url='#{url}'
           if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == resource.pkgdir
             unlocated = ([m["TOPLEVEL"],m["SUBPATH"],m["REST"]] * "/")
             unlocated.gsub!(/\/+/,'/')
-            unlocated[self.subdir] = "" if self.subdir
+            if self.subdir && ! self.subdir.empty?
+              subdir = self.subdir
+              subdir += "/" unless subdir.end_with?("/")
+              unlocated[subdir] = ""
+            end
             choices << self.annotate(unlocated)
           end
         end

data/lib/rbbt/tsv/util.rb

@@ -400,5 +400,9 @@ module TSV
     end
     index
   end
+
+  def merge(other)
+    self.annotate(super(other))
+  end
 end
 

data/lib/rbbt/util/R/plot.rb

@@ -1,6 +1,91 @@
 module R
   module SVG
 
+    def self.plot(filename, data = nil, script = nil, width = nil, height = nil, options = {}, &block)
+      width ||= 600
+      height ||= 600
+      values = []
+
+      script ||= ""
+      if block_given?
+        s = StringIO.new
+        class << s
+          def method_missing(name, *args)
+            name = name.to_s
+            if name[-1] == '='
+              arg = args.first
+              value = if String === arg
+                        arg
+                      else
+                        R.ruby2R arg
+                      end
+              add("" << name[0..-2] << "=" << value)
+            else
+              args_strs = []
+              args.each do |arg|
+                value = if String === arg
+                          arg
+                        else
+                          R.ruby2R arg
+                        end
+                args_strs << value
+              end
+              add("" << name << "(" << args_strs * ", " << ")")
+            end
+          end
+
+          def add(line)
+            self.write line << "\n"
+          end
+        end
+        block.call(s)
+        s.rewind
+        script << "\n" << s.read
+      end
+      sources = [:plot, options[:source]].flatten.compact
+      
+      if data
+        data.each do |k,v|
+          v = Array === v ? v : [v]
+          next if v == "NA" or v.nil? or v.include? "NA" or v.include? nil
+          values = v
+          break
+        end 
+
+        values = [values] unless values.nil? or Array === values
+
+        field_classes = values.collect do |v| 
+          case v
+          when FalseClass, TrueClass
+            "'logical'"
+          when Numeric
+            "'numeric'"
+          when String
+            if v.strip =~ /^[-+]?[\d\.]+$/
+              "'numeric'"
+            else
+              "'character'"
+            end
+          when Symbol
+            "'factor'"
+          else
+            ":NA"
+          end
+        end
+
+        options[:R_open] ||= "colClasses=c('character'," + field_classes * ", " + ')' if field_classes.any?
+
+        data.R <<-EOF, :plot, options
+  rbbt.svg_plot("#{ filename }", width=#{ width }, height = #{ height }, function(){ #{script} })
+  data = NULL
+        EOF
+      else
+        R.run <<-EOF, :plot, options
+  rbbt.svg_plot("#{ filename }", width=#{ width }, height = #{ height }, function(){ #{script} })
+        EOF
+      end
+    end
+
     def self.ggplotSVG(*args)
       ggplot(*args)
     end

data/lib/rbbt/util/cmd.rb

@@ -22,13 +22,17 @@ module CMD
     return tool.to_s unless TOOLS[tool]
 
     @@init_cmd_tool ||= IndiferentHash.setup({})
+
+    claim, test, block, cmd = TOOLS[tool]
+    cmd = tool.to_s if cmd.nil?
+
     if !@@init_cmd_tool[tool]
-      claim, test, block, cmd = TOOLS[tool]
+
       begin
         if test
           CMD.cmd(test + " ")
         else
-          CMD.cmd("#{tool} --help")
+          CMD.cmd("#{cmd} --help")
         end
       rescue
         if claim
@@ -41,7 +45,7 @@ module CMD
       version = nil
       ["--version", "-version", "--help", ""].each do |f|
         begin
-          version_txt += CMD.cmd("#{tool} #{f} 2>&1", :nofail => true).read
+          version_txt += CMD.cmd("#{cmd} #{f} 2>&1", :nofail => true).read
           version = Misc.scan_version_text(version_txt, tool)
           break if version
         rescue
@@ -54,7 +58,7 @@ module CMD
       return cmd if cmd
     end
 
-    tool.to_s
+    cmd
   end
 
   def self.versions

data/lib/rbbt/util/concurrency/processes.rb

@@ -245,7 +245,7 @@ class RbbtProcessQueue
         end
       end if @callback_queue
 
-      @callback_thread.report_on_exception = false
+      @callback_thread.report_on_exception = false if @callback_thread
       
     end
 

data/lib/rbbt/util/migrate.rb

@@ -1,6 +1,5 @@
 module Rbbt 
 
-  prepare_for_execution(job)
   def self.migrate_source_paths(path, resource = Rbbt, source = nil)
     if source
       lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
@@ -60,7 +59,7 @@ puts resource[path].find(search_path)
     test_str = options[:test] ? '-nv' : ''
 
     real_paths.each do |source_path|
-      Log.medium "Migrating #{source_path} #{options[:files].length} files to #{target} - #{Misc.fingerprint(options[:files])}}" if options[:files]
+      Log.low "Migrating #{source_path} #{options[:files].length} files to #{target} - #{Misc.fingerprint(options[:files])}}" if options[:files]
       if File.directory?(source_path) || source_path.end_with?("/")
         source_path += "/" unless source_path.end_with? '/'
         target += "/" unless target.end_with? '/'

data/lib/rbbt/util/misc/inspect.rb

@@ -455,6 +455,7 @@ module Misc
   end
 
   def self.scan_version_text(text, cmd = nil)
+    text = Misc.fixutf8 text
     cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
     text.split("\n").each do |line|
       next unless line =~ /\W#{cmd}\W/i

data/lib/rbbt/util/open.rb

@@ -530,16 +530,22 @@ module Open
 
   def self.gzip?(file)
     file = file.find if Path === file
+    file = file.filename if File === file
+    return false unless String === file
     !! (file =~ /\.gz$/)
   end
 
   def self.bgzip?(file)
     file = file.find if Path === file
+    file = file.filename if File === file
+    return false unless String === file
     !! (file =~ /\.bgz$/)
   end
 
   def self.zip?(file)
     file = file.find if Path === file
+    file = file.filename if File === file
+    return false unless String === file
     !! (file =~ /\.zip$/)
   end
 

data/lib/rbbt/util/ssh.rb

@@ -9,7 +9,7 @@ class SSHLine
     @ssh = Net::SSH.start(@host, @user)
 
     @ch = @ssh.open_channel do |ch|
-      ch.exec 'bash'
+      ch.exec 'bash -l'
     end
 
     @ch.on_data do |_,data|

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -253,7 +253,7 @@ job.clean
       Step.wait_for_jobs missing_deps
 
       migrate_dependencies = all_deps.keys.collect{|d| [d] + d.rec_dependencies + d.input_dependencies }.flatten.select{|d| d.done? }.collect{|d| d.path }
-      Log.medium "Migrating #{migrate_dependencies.length} dependencies from #{Misc.fingerprint job_list} to #{ server }" 
+      Log.low "Migrating #{migrate_dependencies.length} dependencies from #{Misc.fingerprint job_list} to #{ server }" 
       Step.migrate(migrate_dependencies, search_path, :target => server) if migrate_dependencies.any?
     end
 

data/lib/rbbt/workflow/step/dependencies.rb

@@ -145,9 +145,9 @@ class Step
   end
 
   def input_dependencies
-    @input_dependencies ||= recursive_inputs.flatten.
+    @input_dependencies ||= recursive_inputs(true).flatten.
       select{|i| Step === i || (defined?(RemoteStep) && RemoteStep === i) } + 
-      recursive_inputs.flatten.
+      recursive_inputs(true).flatten.
       select{|dep| Path === dep && Step === dep.resource }.
       #select{|dep| ! dep.resource.started? }. # Ignore input_deps already started
       collect{|dep| dep.resource }
@@ -581,7 +581,6 @@ class Step
   #but dependencies are absent, meanining that the file could have been dropped
   #in
   def rec_dependencies(connected = false, seen = [])
-
     # A step result with no info_file means that it was manually
     # placed. In that case, do not consider its dependencies
     return [] if ! (defined? WorkflowRemoteClient && WorkflowRemoteClient::RemoteStep === self) && ! Open.exists?(self.info_file) && Open.exists?(self.path.to_s) 
@@ -599,7 +598,7 @@ class Step
       #next if self.done? && Open.exists?(info_file) && info[:dependencies] && info[:dependencies].select{|task,name,path| path == step.path }.empty?
       next if archived_deps.include? step.path
       next if seen.include? step
-      next if self.done? && connected && ! updatable?
+      next if step.done? && connected && ! step.updatable?
 
       r = step.rec_dependencies(connected, new_dependencies)
       new_dependencies.concat r

data/lib/rbbt/workflow/step.rb

@@ -90,7 +90,7 @@ class Step
     end
   end
 
-  def recursive_inputs
+  def recursive_inputs(connected = false)
     if NamedArray === inputs
       i = {}
       inputs.zip(inputs.fields).each do |v,f|
@@ -99,7 +99,7 @@ class Step
     else
       i = {}
     end
-    rec_dependencies.each do |dep|
+    rec_dependencies(connected).each do |dep|
       next unless NamedArray === dep.inputs
 
       dep.inputs.zip(dep.inputs.fields).each do |v,f|

data/lib/rbbt/workflow/usage.rb

@@ -20,7 +20,7 @@ module Task
       task_inputs = dep_inputs deps, workflow
       task_inputs.each do |task,new_inputs|
         new_inputs.zip(task.input_types.values_at(*new_inputs)).select do |i,t| 
-          t.to_sym == :select and task.input_options[i][:select_options] 
+          t.to_sym == :select && task.input_options.include?(i) && task.input_options[i][:select_options] 
         end.each do |i,t| 
           selects << [i, task.input_options[i][:select_options]] 
         end

data/lib/rbbt/workflow/util/trace.rb

@@ -81,7 +81,8 @@ module Workflow
 
   def self.plot_trace_job_times(data, plot, width=800, height=800)
     data.R <<-EOF, [:svg]
-rbbt.require('tidyverse')
+rbbt.require('dplyr')
+rbbt.require('tidyr')
 rbbt.require('ggplot2')
 
 names(data) <- make.names(names(data))