rbbt-util 5.23.18 → 5.23.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 39526da82633a38054bbce4d640d5006f0e2f3d2
-  data.tar.gz: 2bb4766c1d2aea82fa9cdbccfa4ebc8f6ef49deb
+  metadata.gz: 5ea281c4b995f87831e4ad034423d3ca5ff1b6f5
+  data.tar.gz: 4e8c74787a866cbfdf0bc0b45d18b89f043751a2
 SHA512:
-  metadata.gz: bc8a05f50a9615be78b3cf65cac028208e02d8a00b0fdca73ddd389258e1e836cc573d25dcf90fb45805a322a204794f137ebb58f270f351c8646c6e2ec014f7
-  data.tar.gz: 1a9a0b9baf29b552cf4d19f0343d1154dccc87ecbe522226f49c6090b6c631ae1680505eedaa483aa117020583562679c3211c5ad5c4b0376ed559e187e5cb03
+  metadata.gz: 6c62ba3da73a0cf27a1aa41cb82b6b9d32a97dac3dafe62efbd7552a7a6cc58f69b577708b377f1b34f6b50d748a06e95b8e34e2b95b69a1fb392a1292adedff
+  data.tar.gz: 74c2fbcee1ada86a02a540232bec94581048b585dcb55015ae59b1988b9d0de8e4b460bffabf5100f3b043486c35f04b2b430da1c39521332d3c1d3a7360946a

data/lib/rbbt/persist.rb

@@ -371,7 +371,7 @@ module Persist
       when type.to_sym == :memory
         repo = persist_options[:repo] || Persist::MEMORY
         path = path.find if Path === path
-        repo[path] = yield
+        repo[path] ||= yield
 
       when (type.to_sym == :annotations and persist_options.include? :annotation_repo)
 
@@ -468,6 +468,7 @@ module Persist
     when String
       persist name, :memory, :file => name + "_" << options, &block
     else
+      options = options.dup
       file = name
       repo = options.delete :repo if options and options.any?
       file << "_" << (options[:key] ? options[:key] : Misc.hash2md5(options)) if options and options.any?

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -195,7 +195,7 @@ class RbbtProcessQueue
 
       @pid = Process.fork do
         Misc.pre_fork
-        Log::ProgressBar.add_offset if @offset
+        Log::ProgressBar.add_offset @offset if @offset
 
         @cleanup.call if @cleanup
         @queue.close_write 
@@ -212,7 +212,7 @@ class RbbtProcessQueue
         else
           run
         end
-        Log::ProgressBar.remove_offset if @offset
+        Log::ProgressBar.remove_offset @offset if @offset
       end
     end
 

data/lib/rbbt/util/concurrency/processes.rb

@@ -118,16 +118,17 @@ class RbbtProcessQueue
                 while @count > 0
                   @count -= 1
                   @total += 1
-                  processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, @offset, &@init_block)
+                  processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, (@offset ? @total : false), &@init_block)
                   Log.warn "Added process #{processes.last.pid} to #{Process.pid} (#{processes.length})"
                 end
 
                 while @count < 0
                   @count += 1
+                  @total -= 1
                   next unless processes.length > 1
-                  first = processes.shift
-                  first.stop
-                  Log.warn "Removed process #{first.pid} from #{Process.pid} (#{processes.length})"
+                  last = processes.last
+                  last.stop
+                  Log.warn "Removed process #{last.pid} from #{Process.pid} (#{processes.length})"
                 end
               end
             end
@@ -148,7 +149,7 @@ class RbbtProcessQueue
       num_processes.times do |i|
         @total += 1 
         process_mutex.synchronize do
-          processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, @offset, &@init_block)
+          processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, (@offset ? @total : false), &@init_block)
         end
       end
 
@@ -189,6 +190,7 @@ class RbbtProcessQueue
 
       RbbtSemaphore.post_semaphore(@sem)
 
+      Log.low "Process monitor #{Process.pid} joining threads"
       begin
         @monitor_thread.join
         @manager_thread.raise TryAgain if @manager_thread.alive?
@@ -197,6 +199,7 @@ class RbbtProcessQueue
       rescue Exception
         Kernel.exit -1
       end
+      Log.low "Process monitor #{Process.pid} threads joined successfully, now exit"
 
       Kernel.exit 0
     end

data/lib/rbbt/util/log/progress/report.rb

@@ -111,15 +111,19 @@ module Log
       if short_mean
         thr = short_mean
       else
-        thr = @ticks / (Time.now - @start) 
+        thr = begin
+                (@ticks || 1) / (Time.now - @start) 
+              rescue
+                1
+              end
       end
       
       if mean.nil? or mean.to_i > 1
         str = "#{ Log.color :blue, thr.to_i.to_s } per sec."
-        str << " #{ Log.color :yellow, mean.to_i.to_s } avg. #{Log.color :yellow, @mean_max.to_i.to_s} max." if @mean_max > 0
+        #str << " #{ Log.color :yellow, mean.to_i.to_s } avg. #{Log.color :yellow, @mean_max.to_i.to_s} max." if @mean_max > 0
       else
         str = "#{ Log.color :blue, (1/thr).ceil.to_s } secs each"
-        str << " #{ Log.color :yellow, (1/mean).ceil.to_s } avg. #{Log.color :yellow, (1/@mean_max).ceil.to_s} min." if @mean_max > 0
+        #str << " #{ Log.color :yellow, (1/mean).ceil.to_s } avg. #{Log.color :yellow, (1/@mean_max).ceil.to_s} min." if @mean_max > 0
       end
 
       str
@@ -158,12 +162,12 @@ module Log
     end
 
     def report_msg
-      str = Log.color :magenta, desc
+      str = Log.color(:magenta, "·")
       if @ticks == 0
         if @max
-          return str << " " << Log.color(:yellow, "waiting on #{@max} #{bytes ? 'bytes' : 'items'}") 
+          return str << " " << Log.color(:magenta, "waiting on #{@max} #{bytes ? 'bytes' : 'items'}") <<  Log.color(:magenta, " · " << desc)
         else
-          return str << " " << Log.color(:yellow, "waiting - PID: #{Process.pid}") 
+          return str << " " << Log.color(:magenta, "waiting - PID: #{Process.pid}") <<  Log.color(:magenta, " · " << desc)
         end
       end
       str << " " << thr_msg
@@ -172,6 +176,7 @@ module Log
       else
         str << Log.color(:blue, " -- ") << ticks.to_s << " #{bytes ? 'bytes' : 'items'}"
       end
+      str <<  Log.color(:magenta, " · " << desc)
       str
     end
 
@@ -183,14 +188,20 @@ module Log
 
     def report(io = STDERR)
       if Log::LAST != "progress"
-        length = Log::ProgressBar.cleanup_bars
         bars = BARS
-        print(io, Log.color(:yellow, "...Progress\n"))
-        bars.sort_by{|b| b.depth }.reverse.each do |bar|
-          if SILENCED.include? bar
-            print(io, Log.color(:yellow ,bar.report_msg) << "\n") 
-          else
-            print(io, "\n") 
+        if Log::LAST == "new_bar"
+          Log::LAST.replace "progress"
+          bar = bars.sort_by{|b| b.depth }.first
+          print(io, Log.color(:magenta ,bar.report_msg) << "\n") 
+        else
+          length = Log::ProgressBar.cleanup_bars
+          print(io, Log.color(:magenta, "···Progress\n"))
+          bars.sort_by{|b| b.depth }.reverse.each do |bar|
+            if SILENCED.include? bar
+              print(io, Log.color(:magenta, "·\n")) 
+            else
+              print(io, Log.color(:magenta ,bar.report_msg) << "\n") 
+            end
           end
         end
       else
@@ -198,7 +209,7 @@ module Log
       end
       bars << self unless BARS.include? self
 
-      print(io, Log.up_lines(bars.length+1) << Log.color(:yellow, "...Progress\n") << Log.down_lines(bars.length+1)) 
+      print(io, Log.up_lines(bars.length) << Log.color(:magenta, "···Progress\n") << Log.down_lines(bars.length+1)) if Log::ProgressBar.offset == 0
       print(io, Log.up_lines(@depth) << report_msg << Log.down_lines(@depth)) 
       @last_time = Time.now
       @last_count = ticks
@@ -207,7 +218,7 @@ module Log
     end
 
     def done(io = STDERR)
-      done_msg = Log.color(:magenta, desc) << " " << Log.color(:green, "done")
+      done_msg = Log.color(:magenta, "· ") << Log.color(:green, "done")
       if @start
         ellapsed = (Time.now - @start).to_i
       else
@@ -217,13 +228,14 @@ module Log
       done_msg << " " << Log.color(:blue, (@ticks).to_s) << " #{bytes ? 'bytes' : 'items'} in " << Log.color(:green, ellapsed)
       @last_count = 0
       @last_time = @start
-      done_msg << " (" << thr_msg << ")"
+      done_msg << " - " << thr_msg 
+      done_msg << Log.color(:magenta, " · " << desc)
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
       Open.rm @file if @file and Open.exists? @file
     end
 
     def error(io = STDERR)
-      done_msg = Log.color(:magenta, desc) << " " << Log.color(:red, "error")
+      done_msg = Log.color(:magenta, "· ") << Log.color(:red, "error")
       if @start
         ellapsed = (Time.now - @start).to_i
       else
@@ -233,7 +245,8 @@ module Log
       done_msg << " " << Log.color(:blue, (@ticks).to_s) << " in " << Log.color(:green, ellapsed)
       @last_count = 0
       @last_time = @start
-      done_msg << " (" << thr_msg << ")"
+      done_msg << " - " << thr_msg
+      done_msg << Log.color(:magenta, " · " << desc)      
       print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth)) 
       Open.rm @file if @file and Open.exists? @file
     end

data/lib/rbbt/util/log/progress/util.rb

@@ -5,14 +5,16 @@ module Log
     REMOVE = []
     SILENCED = []
 
-    def self.add_offset
-      @@offset = offset + 1
+    def self.add_offset(value = 1)
+      value = 1 if TrueClass === value
+      @@offset = offset + value.to_i
       @@offset = 0 if @@offset < 0
       @@offset
     end
 
-    def self.remove_offset
-      @@offset = offset - 1
+    def self.remove_offset(value = 1)
+      value = 1 if TrueClass === value
+      @@offset = offset - value.to_i
       @@offset = 0 if @@offset < 0
       @@offset
     end
@@ -27,7 +29,7 @@ module Log
     def self.new_bar(max, options = {})
       cleanup_bars
       BAR_MUTEX.synchronize do
-        #Log::LAST.replace "new_bar" if Log::LAST == "progress"
+        Log::LAST.replace "new_bar" if Log::LAST == "progress"
         options = Misc.add_defaults options, :depth => BARS.length + Log::ProgressBar.offset
         BARS << (bar = ProgressBar.new(max, options))
         bar
@@ -69,6 +71,7 @@ module Log
       BAR_MUTEX.synchronize do
         REMOVE << bar
       end
+      Log::LAST.replace "remove_bar" if Log::LAST == "progress"
     end
 
     def self.with_bar(max, options = {})

data/lib/rbbt/util/log/progress.rb

@@ -6,7 +6,7 @@ module Log
 
     attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes
     def initialize(max = nil, options = {})
-      options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :desc => "Progress", :io => STDERR, :severity => Log.severity, :frequency => 2
+      options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :io => STDERR, :severity => Log.severity, :frequency => 2
       depth, num_reports, desc, io, severity, file, bytes, frequency = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency
 
       @max = max
@@ -16,7 +16,7 @@ module Log
       @last_count = nil
       @last_percent = nil
       @depth = depth
-      @desc = desc.nil? ? "Progress" : desc.gsub(/\n/,' ')
+      @desc = desc.nil? ? "" : desc.gsub(/\n/,' ')
       @file = file
       @bytes = bytes
     end

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -85,7 +85,7 @@ module ConcurrentStream
             Log.low "Not failing on exception joining thread in ConcurrenStream: #{filename}"
           else
             Log.low "Exception joining thread in ConcurrenStream: #{filename}"
-            raise $! 
+            stream_raise_exception $! 
           end
         end
       end
@@ -98,7 +98,7 @@ module ConcurrentStream
       @pids.each do |pid| 
         begin
           Process.waitpid(pid, Process::WUNTRACED)
-          raise ProcessFailed.new "Error joining process #{pid} in #{self.filename || self.inspect}" unless $?.success? or no_fail
+          stream_raise_exception ProcessFailed.new "Error joining process #{pid} in #{self.filename || self.inspect}" unless $?.success? or no_fail
         rescue Errno::ECHILD
         end
       end 
@@ -203,7 +203,7 @@ module ConcurrentStream
         Log.exception $!
         self.abort
         self.join 
-        raise $!
+        stream_raise_exception $!
       ensure
         self.join if self.closed? or self.eof? 
       end
@@ -220,16 +220,12 @@ module ConcurrentStream
     end
   end
 
-  def raise(exception)
-    begin
-      threads.each do |thread|
-        thread.raise exception
-      end
-
-      self.abort
-    ensure
-      Kernel.raise exception
+  def stream_raise_exception(exception)
+    threads.each do |thread|
+      thread.raise exception
     end
+
+    self.abort
   end
 
 end

data/lib/rbbt/util/misc/system.rb

@@ -45,6 +45,18 @@ end
       end
   end
 
+  def self.with_env(var, value, &block)
+    var = var.to_s
+    value = value.to_s
+    current = ENV[var]
+    begin
+      ENV[var] = value
+      yield
+    ensure
+      ENV[var] = current
+    end
+  end
+
   def self.path_relative_to(basedir, path)
     path = File.expand_path(path) unless path[0] == "/"
     basedir = File.expand_path(basedir) unless basedir[0] == "/"

data/lib/rbbt/workflow/accessor.rb

@@ -106,18 +106,19 @@ class Step
   def info_lock
     @info_lock = begin
                    path = Persist.persistence_path(info_file + '.lock', {:dir => Step.lock_dir})
-                   Lockfile.new path, :refresh => false, :dont_use_lock_id => true
+                   #Lockfile.new path, :refresh => false, :dont_use_lock_id => true
+                   Lockfile.new path
                  end if @info_lock.nil?
     @info_lock
   end
 
   def status_lock
     return @mutex
-    @status_lock = begin
-                   path = Persist.persistence_path(info_file + '.status.lock', {:dir => Step.lock_dir})
-                   Lockfile.new path, :refresh => false, :dont_use_lock_id => true
-                 end if @status_lock.nil?
-    @status_lock
+    #@status_lock = begin
+    #               path = Persist.persistence_path(info_file + '.status.lock', {:dir => Step.lock_dir})
+    #               Lockfile.new path, :refresh => false, :dont_use_lock_id => true
+    #             end if @status_lock.nil?
+    #@status_lock
   end
 
   def info(check_lock = true)
@@ -949,7 +950,7 @@ module Workflow
                      _inputs = assign_dep_inputs(_inputs, options, all_d, workflow.task_info(dep_task))
                      jobname = _inputs[:jobname] if _inputs.include? :jobname
 
-                     job = workflow.job(dep_task, jobname, _inputs)
+                     job = workflow._job(dep_task, jobname, _inputs)
                      ComputeDependency.setup(job, compute) if compute
                      job
                    end
@@ -963,7 +964,7 @@ module Workflow
                      d_.overriden = true
                      d_
                    else
-                     job(dependency, jobname, _inputs)
+                     _job(dependency, jobname, _inputs)
                    end
                  when Proc
                    if DependencyBlock === dependency
@@ -994,7 +995,7 @@ module Workflow
                            task_info = d[:workflow].task_info(d[:task])
 
                            inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
-                           d = d[:workflow].job(d[:task], d[:jobname], inputs) 
+                           d = d[:workflow]._job(d[:task], d[:jobname], inputs) 
                          end
                        end
                        ComputeDependency.setup(d, compute) if compute
@@ -1014,7 +1015,7 @@ module Workflow
                        else
                          task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
                          inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
-                         dep = dep[:workflow].job(dep[:task], dep[:jobname], inputs)
+                         dep = dep[:workflow]._job(dep[:task], dep[:jobname], inputs)
                        end
                      end
                    end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -236,7 +236,7 @@ class Step
     when :bootstrap
       cpus = rest.nil? ? nil : rest.first 
 
-      cpus = config('dep_cpus', 'bootstrap', :default => [5, list.length / 2].min) if cpus.nil?
+      cpus = config('dep_cpus', 'bootstrap', :default => [5, list.length / 2].min) if cpus.nil? || cpus.to_i == 0
 
       respawn = rest && rest.include?(:respawn)
       respawn = false if rest && rest.include?(:norespawn)
@@ -256,9 +256,13 @@ class Step
               Log.warn "Error in bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
 
             rescue Aborted
-              dep.abort
-              Log.warn "Aborted bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
-              raise $!
+              ex = $!
+              begin
+                dep.abort
+                Log.warn "Aborted bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
+              rescue
+              end
+              raise StopInsist.new(ex)
 
             rescue RbbtException
               if canfail || dep.canfail?
@@ -267,7 +271,10 @@ class Step
                 Log.warn "NOT Allowing failing of #{dep.path}: #{dep.messages.last}"
                 dep.exception $!
                 if dep.recoverable_error?
-                  dep.abort
+                  begin
+                    dep.abort
+                  rescue
+                  end
                   raise $!
                 else
                   raise StopInsist.new($!)
@@ -290,6 +297,7 @@ class Step
 
   def canfail_paths
     return Set.new if ! File.exists?(info_file)
+
     if info[:canfail_paths]
       Set.new(info[:canfail_paths])
     else

data/lib/rbbt/workflow/step/run.rb

@@ -104,22 +104,36 @@ class Step
     (no_load or ENV["RBBT_NO_STREAM"]) ? @result : prepare_result(@result, @task.result_description)
   end
 
+  def updatable?
+    (ENV["RBBT_UPDATE_ALL_JOBS"] == 'true' || Open.exists?(info_file)) && ! relocated?
+  end
+
   def dependency_checks
     rec_dependencies.
       select{|dependency| ! (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency) }.
       select{|dependency| ! Open.remote?(dependency.path) }.
-      select{|dependency| Open.exists?(dependency.info_file) && ! dependency.relocated? }.
+      select{|dependency| dependency.updatable? }.
       select{|dependency| ! dependency.error? }
   end
 
   def input_checks
-    inputs.select{|i| Step === i }
+    inputs.select{|i| Step === i }.
+      select{|dependency| dependency.updatable? }
   end
 
   def checks
     (dependency_checks + input_checks).uniq
   end
 
+  def persist_checks
+    canfail_paths = self.canfail_paths
+    checks.collect do |dep| 
+      path = dep.path
+      next if ! dep.done? && canfail_paths.include?(path)
+      path 
+    end.compact
+  end
+
   def out_of_date
 
     checks = self.checks
@@ -128,8 +142,7 @@ class Step
     outdated_dep  = []
     canfail_paths = self.canfail_paths
     checks.each do |dep| 
-      next unless Open.exists?(dep.info_file)
-      next if dep.relocated?
+      next unless dep.updatable?
 
       begin
         if dep.done? && self.done? && Open.exists?(dep.path) && Open.exists?(self.path) && (File.mtime(dep.path) > File.mtime(self.path))
@@ -183,7 +196,7 @@ class Step
         no_load = :stream if no_load
 
         Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
-        result = Persist.persist "Job", @task.result_type, :file => path, :check => checks.collect{|dep| dep.path}, :no_load => no_load do 
+        result = Persist.persist "Job", @task.result_type, :file => path, :check => persist_checks, :no_load => no_load do 
           if Step === Step.log_relay_step and not self == Step.log_relay_step
             relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
           end

data/lib/rbbt/workflow/step.rb

@@ -310,7 +310,7 @@ class Step
     self
   end
 
-  def rec_dependencies(need_run = false)
+  def rec_dependencies(need_run = false, seen = [])
 
     # A step result with no info_file means that it was manually
     # placed. In that case, do not consider its dependencies
@@ -320,9 +320,10 @@ class Step
 
     new_dependencies = []
     dependencies.each{|step| 
-      next if self.done? && need_run && Open.exists?(step.info_file) 
+      next if seen.include? step
+      next if self.done? && need_run && ! updatable?
 
-      r = step.rec_dependencies(need_run)
+      r = step.rec_dependencies(need_run, new_dependencies)
       new_dependencies.concat r
       new_dependencies << step
     }

data/lib/rbbt/workflow.rb

@@ -295,11 +295,10 @@ module Workflow
     @_m
   end
 
-  def job(taskname, jobname = nil, inputs = {})
+  def __job(taskname, jobname = nil, inputs = {})
     taskname = taskname.to_sym
     return remote_tasks[taskname].job(taskname, jobname, inputs) if remote_tasks and remote_tasks.include? taskname
 
-
     task = tasks[taskname]
     raise "Task not found: #{ taskname }" if task.nil?
 
@@ -308,8 +307,9 @@ module Workflow
     Workflow.resolve_locals(inputs)
 
     task_info = task_info(taskname)
-    task_inputs = task.inputs
+    task_inputs = task_info[:inputs]
     #defaults = IndiferentHash.setup(task_info[:input_defaults]).merge(task.input_defaults)
+    all_defaults = IndiferentHash.setup(task_info[:input_defaults])
     defaults = IndiferentHash.setup(task.input_defaults)
 
     missing_inputs = []
@@ -339,14 +339,14 @@ module Workflow
 
     inputs.each do |k,v|
       next unless (task_inputs.include?(k.to_sym) or task_inputs.include?(k.to_s))
-      default = defaults[k]
+      default = all_defaults[k]
       next if default == v 
       next if (String === default and Symbol === v and v.to_s == default)
       next if (Symbol === default and String === v and v == default.to_s)
       real_inputs[k] = v 
     end
 
-    jobname_input_value = inputs[jobname_input] || defaults[jobname_input]
+    jobname_input_value = inputs[jobname_input] || all_defaults[jobname_input]
     if jobname_input && jobname.nil? && String === jobname_input_value && ! jobname_input_value.include?('/')
       jobname = jobname_input_value
     end
@@ -371,6 +371,26 @@ module Workflow
     job
   end
 
+  def _job(taskname, jobname = nil, inputs = {})
+
+    _inputs = IndiferentHash.setup(inputs.dup)
+
+    task_info = task_info(taskname)
+    task_inputs = task_info[:inputs]
+    Persist.memory("STEP", :taskname => taskname, :jobname => jobname, :inputs => inputs.values_at(*task_inputs), :repo => step_cache) do
+      __job(taskname, jobname, inputs)
+    end
+  end
+
+  def job(taskname, jobname = nil, inputs = {})
+    begin
+      _job(taskname, jobname, inputs)
+    ensure
+      step_cache.clear
+    end
+  end
+
+
   def set_step_dependencies(step)
     if step.info[:dependencies]
       Misc.insist do

data/share/install/software/lib/install_helpers

@@ -10,6 +10,16 @@ OPT_BUILD_DIR="$SOFTWARE_DIR/.build"; [ -d $OPT_BUILD_DIR ] || mkdir -p $OPT_BUI
 
 mkdir -p "$OPT_BUILD_DIR"
 
+function link(){
+  local source="$1"
+  local target="$2"
+  local rel_source=$(realpath --relative-to="$(dirname $target)" "$source")
+  
+  [ -h "$target" ] && rm "$target"
+  echo ln -s "$rel_source" "$target"
+  ln -s "$rel_source" "$target"
+}
+
 function expand_path(){
   name=$(basename $1)
   dir=$(dirname $1)
@@ -77,7 +87,7 @@ get_svn(){
     clean_build
 
     cd $OPT_BUILD_DIR
-    ln -s "$OPT_SCM_DIR/$name" "$name"
+    link "$OPT_SCM_DIR/$name" "$name"
 
     cd "$old_pwd"
 
@@ -107,7 +117,7 @@ get_git(){
     clean_build
 
     cd $OPT_BUILD_DIR
-    ln -s "$OPT_SCM_DIR/$name" "$name"
+    link "$OPT_SCM_DIR/$name" "$name"
 
     cd "$old_pwd"
 }
@@ -204,10 +214,17 @@ opt_dir(){
 
 move_opt(){
     local name="$1"
-    local pkg_dir="`opt_dir \"$name\"`"
+    local pkg_dir=$(opt_dir "$name")
+    local source=$(build_dir)
 
     mkdir -p $(dirname "$pkg_dir")
-    mv "`build_dir`" "$pkg_dir"
+    if [ -h "$source" ]; then
+      local real_source=$(realpath "$source")
+      echo link "$real_source" "$pkg_dir"
+      link "$real_source" "$pkg_dir"
+    else
+      mv "$source" "$pkg_dir"
+    fi
 }
 
 setup(){
@@ -216,7 +233,7 @@ setup(){
 
     if versioned? "$name"; then
         rm -f "$(dirname $pkg_dir)/current"
-        ln -sf "$pkg_dir" "$(dirname $pkg_dir)/current"
+        link "$pkg_dir" "$(dirname $pkg_dir)/current"
         pkg_dir="$(dirname $pkg_dir)/current"
     fi
 
@@ -227,7 +244,7 @@ setup(){
         for exe in ` find "$pkg_dir/bin/" -maxdepth 1 -type f -executable`; do
             exe=$(basename $exe)
             rm -f ./$exe
-            ln -s "$pkg_dir/bin/$exe" . 2>/dev/null
+            link "$pkg_dir/bin/$exe" . 2>/dev/null
         done
 
         cd "$old_pwd"
@@ -238,7 +255,7 @@ setup(){
         for exe in ` find "$pkg_dir/" -maxdepth 1 -type f -executable`; do
             exe=$(basename $exe)
             rm -f ./$exe
-            ln -s "$pkg_dir/$exe" . 2>/dev/null
+            link "$pkg_dir/$exe" . 2>/dev/null
         done
 
         cd "$old_pwd"
@@ -298,7 +315,7 @@ install_jar(){
 
     [ -d  "$OPT_DIR/$name/" ] || mkdir -p  "$OPT_DIR/$name/"
     wget "$url" -O "$OPT_DIR/$name/$name.jar"
-    ln -s "$OPT_DIR/$name/$name.jar" "$OPT_JAR_DIR/$name.jar" 
+    link "$OPT_DIR/$name/$name.jar" "$OPT_JAR_DIR/$name.jar" 
 }
 
 jar2bin(){

data/share/rbbt_commands/workflow/task

@@ -190,6 +190,7 @@ the job dependencies recursively.
 -cl--clean Clean the last step of the job so that it gets recomputed
 -ct--clean_task* Clean a particular dependency task
 -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely
+-uaj--update_all_jobs Consider all dependencies when checking for updates, even when they have no info files
 --fork Run job asyncronously and monitor progress. It monitors detached processes as well
 --detach Run job asyncronously and detach process
 --exec Run job with no persistence
@@ -234,6 +235,10 @@ end
 
 workflow = Workflow.require_workflow workflow
 
+if options[:update_all_jobs]
+  ENV["RBBT_UPDATE_ALL_JOBS"] = 'true'
+end
+
 if options[:workflows]
   require 'rbbt/workflow'
   workflows = options[:workflows].split(',')
@@ -285,11 +290,11 @@ name = options.delete(:jobname)
 
 # get job args
 sopt_option_string = SOPT_options(workflow, task)
+job_options = SOPT.get sopt_option_string
+
 if options[:load_inputs]
   task_info = workflow.task_info(task_name)
-  job_options = Workflow.load_inputs(options[:load_inputs], task_info[:inputs], task_info[:input_types]).merge(SOPT.get(sopt_option_string))
-else
-  job_options = SOPT.get sopt_option_string
+  job_options = Workflow.load_inputs(options[:load_inputs], task_info[:inputs], task_info[:input_types]).merge(SOPT.get(sopt_option_string)).merge(job_options)
 end
 job_options = fix_options(workflow, task, job_options)
 saved_job_options = job_options

data/test/rbbt/test_workflow.rb

@@ -304,4 +304,21 @@ class TestWorkflow < Test::Unit::TestCase
     assert_equal real_other, Workflow.relocate(real, other)
   end
 
+  def test_delete_dep
+    job = TestWF.job(:t3).recursive_clean
+    job.run
+    assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
+    job = TestWF.job(:t3)
+    job.step(:t1).clean
+    assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
+    job = TestWF.job(:t3).recursive_clean
+    job.run
+    assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
+    job = TestWF.job(:t3)
+    job.step(:t1).clean
+    Misc.with_env "RBBT_UPDATE_ALL_JOBS", "true" do
+      assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
+    end
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.23.18
+  version: 5.23.19
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-16 00:00:00.000000000 Z
+date: 2018-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake