rbbt-util 5.22.4 → 5.22.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 694be20134e0bd77c1d240c04fe52f547cd4b431
-  data.tar.gz: f96f17c3f62ec9f5d781036966c022f275e8b879
+  metadata.gz: 1c617d4a4ffaed12ddf8a5cdf44a4ff5113e47d0
+  data.tar.gz: 74125cd2876fff8d7c097d89eee4adb47f61a26c
 SHA512:
-  metadata.gz: 6d725866e6cbf97a5a627ca5e575d202e348e2688bf37aff4032d8c752b7fd6e747e1df85c00cf69de6258b7112812da43356db552f65940f22ef86101107afb
-  data.tar.gz: 98dfdd3a29d277cb2c2c52f76e6ffbae1c7f1d033f823a36637235b7b47bb61e0ae43652c14f626a9341df41f370fa0fa7ed55b2a625a63896ffce82c4d9d37a
+  metadata.gz: 81bf1cb9b56203f6ab747ef961fc920146e07b78e2a220fdf2e9174adfd0effbf01974e4560ff6759c2e180d4ce03ea6c2db4fc2a047e07162c342a265735b8f
+  data.tar.gz: c4ef6e26f7683f2365f9ca15cfc045cf69a04a174d2f59a0dce0a22ede010b22f962d17d9bc03f73469338c1c47c2c6cf33a6d65173e387e6f71c633462fabfc

data/lib/rbbt/annotations.rb

@@ -160,7 +160,7 @@ module Annotated
         object
     when Array
       object.respond_to?(:clean_annotations) ?
-        object.clean_annotations : 
+        object.clean_annotations(true) : 
         object.inject([]){|acc,e| acc << Annotated.purge(e); acc}
     when Hash
       new = {}

data/lib/rbbt/monitor.rb

@@ -146,12 +146,13 @@ module Rbbt
           task = File.basename(taskdir)
           next if tasks and not tasks.include? task
 
-          cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/*.files\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' -not -type d 2>/dev/null"
+          #cmd = "find -L '#{ taskdir }/'  -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' 2>/dev/null"
+          cmd = "find -L '#{ taskdir }/' -not \\( -path \"#{taskdir}/*.files/*\" -prune \\) -not -name '*.pid' -not -name '*.notify' -not -name '\\.*' \\( -not -type d -o -name '*.files' \\)  2>/dev/null"
 
           files = CMD.cmd(cmd, :pipe => true)
           TSV.traverse files, :type => :array, :into => jobs, :_bar => "Finding jobs in #{ taskdir }" do |file|
             _files << file
-            if m = file.match(/(.*).(info|pid)$/)
+            if m = file.match(/(.*)\.(info|pid|files)$/)
               file = m[1]
             end
             next if seen.include? file

data/lib/rbbt/resource.rb

@@ -194,10 +194,7 @@ source "$INSTALL_HELPER_FILE"
               EOF
 
               script = preamble + "\n" + Open.read(content)
-              install_io = CMD.cmd('bash', :in => script, :log => true, :pipe => true, :stderr => true)
-              while line = install_io.gets
-                Log.debug line
-              end
+              CMD.cmd_log('bash', :in => script)
 
               set_software_env(software_dir)
             else

data/lib/rbbt/resource/path.rb

@@ -141,7 +141,7 @@ module Path
                      if self.match(/(.*?)\/(.*)/)
                        toplevel, subpath = self.match(/(.*?)\/(.*)/).values_at 1, 2
                      else
-                       toplevel, subpath = self, ""
+                       toplevel, subpath = "{REMOVE}", self
                      end
 
                      path = nil
@@ -199,7 +199,9 @@ module Path
                          sub('{SUBPATH}', subpath).
                          sub('{BASENAME}', File.basename(self)).
                          sub('{PATH}', self).
-                         sub('{LIBDIR}', libdir) #, @pkgdir, @resource, @search_paths
+                         sub('{LIBDIR}', libdir).
+                         sub('{REMOVE}/', '').
+                         sub('{REMOVE}', '')
 
                        path = path + '.gz' if File.exist? path + '.gz'
                        path = path + '.bgz' if File.exist? path + '.bgz'

data/lib/rbbt/util/concurrency/processes.rb

@@ -94,6 +94,7 @@ class RbbtProcessQueue
               Log.debug "Closing up process queue #{Process.pid}"
               @count = 0
               Thread.new do
+                Log.debug "Pushing closed stream #{Process.pid}"
                 while true
                   @queue.push ClosedStream.new unless @queue.cleaned 
                 end unless @processes.empty?
@@ -127,7 +128,7 @@ class RbbtProcessQueue
           rescue TryAgain
             retry
           rescue Aborted
-            Log.low "Closing manager thread"
+            Log.low "Aborting manager thread #{Process.pid}"
             raise Aborted
           rescue Exception
             Log.exception $!

data/lib/rbbt/util/docker.rb

@@ -32,7 +32,7 @@ module Docker
             Open.write(directory[name], obj)
           end
         end
-      end
+      end if job_inputs
     else
       TmpFile.with_file do |tmpfile|
         Path.setup(tmpfile)
@@ -55,12 +55,16 @@ module Docker
               Open.write(tmpfile[name], obj)
             end
           end
-        end
+        end if job_inputs
         pipe = false
       end
 
     end
     cmd = "docker run #{mount_cmd} #{image_cmd} #{cmd}"
-    CMD.cmd(cmd, :log => true, :pipe => pipe)
+    if pipe
+      CMD.cmd(cmd, :log => true, :pipe => true)
+    else
+      CMD.cmd_log(cmd, :log => true)
+    end
   end
 end

data/lib/rbbt/util/misc.rb

@@ -67,6 +67,26 @@ module Misc
   def self.tokenize(str)
     str.scan(/"[^"]*"|'[^']*'|[^"'\s]+/)
   end
+
+  def self.timespan(str, default = "s")
+    tokens = {
+      "s" => (1),
+      "m" => (60),
+      "h" => (60 * 60),
+      "d" => (60 * 60 * 24),
+      "w" => (60 * 60 * 24 * 7),
+    }
+
+    tokens[nil] = tokens[default]
+    tokens[""] = tokens[default]
+    time = 0
+    str.scan(/(\d+)(\w?)/).each do |amount, measure|
+      time += amount.to_i * tokens[measure]
+    end
+    time
+  end
+
+  
 end
 
 module PDF2Text

data/lib/rbbt/util/open.rb

@@ -236,7 +236,52 @@ module Open
     end
   end
 
-  def self.mv(source, target, options)
+  def self.ln_s(source, target, options = {})
+    source = source.find if Path === source
+    target = target.find if Path === target
+
+    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
+    FileUtils.rm target if File.exists?(target)
+    FileUtils.ln_s source, target
+  end
+
+  def self.ln(source, target, options = {})
+    source = source.find if Path === source
+    target = target.find if Path === target
+
+    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
+    FileUtils.rm target if File.exists?(target)
+    FileUtils.ln source, target
+  end
+
+  def self.ln_h(source, target, options = {})
+    source = source.find if Path === source
+    target = target.find if Path === target
+
+    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
+    FileUtils.rm target if File.exists?(target)
+    begin
+      CMD.cmd("ln -L '#{ source }' '#{ target }'")
+    rescue ProcessFailed
+      if $!.message.include? "Invalid cross-device link"
+        Log.debug "Could not hard link #{source} and #{target}: cross-device link"
+        CMD.cmd("cp -L '#{ source }' '#{ target }'")
+      else
+        raise $!
+      end
+    end
+  end
+
+  def self.cp(source, target, options = {})
+    source = source.find if Path === source
+    target = target.find if Path === target
+
+    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
+    FileUtils.rm target if File.exists?(target)
+    FileUtils.cp source, target
+  end
+
+  def self.mv(source, target, options = {})
     dir_sub_path_source = find_repo_dir(source)
     dir_sub_path_target = find_repo_dir(target)
 
@@ -276,7 +321,7 @@ module Open
       exists_in_repo(*dir_sub_path)
     else
       file = file.find if Path === file
-      File.exist? file
+      File.exist?(file) || File.symlink?(file)
     end
   end
 

data/lib/rbbt/workflow.rb

@@ -310,7 +310,8 @@ module Workflow
 
     task_info = task_info(taskname)
     task_inputs = task_info[:inputs]
-    defaults = IndiferentHash.setup(task_info[:input_defaults]).merge(task.input_defaults)
+    #defaults = IndiferentHash.setup(task_info[:input_defaults]).merge(task.input_defaults)
+    defaults = IndiferentHash.setup(task.input_defaults)
 
     missing_inputs = []
     task.required_inputs.each do |input|
@@ -328,7 +329,6 @@ module Workflow
     dependencies = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
 
     real_inputs = {}
-    recursive_inputs = rec_inputs(taskname)
 
     inputs.each do |k,v|
       default = defaults[k]
@@ -339,7 +339,9 @@ module Workflow
       real_inputs[k] = v 
     end
 
-    if real_inputs.empty? and not Workflow::TAG == :inputs
+    overriden = dependencies.select{|dep| dep.overriden }.any?
+
+    if real_inputs.empty? and not Workflow::TAG == :inputs and not overriden 
       step_path = step_path taskname, jobname, [], [], task.extension
       input_values = task.take_input_values(inputs)
     else
@@ -350,6 +352,7 @@ module Workflow
     job = get_job_step step_path, task, input_values, dependencies
     job.workflow = self
     job.clean_name = jobname
+    job.overriden = overriden
     job
   end
 

data/lib/rbbt/workflow/accessor.rb

@@ -378,6 +378,7 @@ class Step
     canfail_paths = self.canfail_paths
     dirty_files = rec_dependencies.reject{|dep|
       (defined?(WorkflowRESTClient) && WorkflowRESTClient::RemoteStep === dep) || 
+        ! Open.exists?(dep.info_file) ||
         (dep.path && (Open.exists?(dep.path) || Open.remote?(dep.path))) || 
         ((dep.error? || dep.aborted?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
     }
@@ -392,6 +393,7 @@ class Step
     if done? and not (status == :done or status == :ending or status == :producing) and not status == :noinfo
       return true 
     end
+
     if status == :done and not done?
       return true 
     end
@@ -561,6 +563,18 @@ class Step
 
     value || default
   end
+
+  def access
+    CMD.cmd("touch -c -h -a #{self.path} #{self.info_file}")
+  end
+  
+  def rec_access
+    access
+    rec_dependencies.each do |dep|
+      dep.access
+    end
+  end
+
 end
 
 module Workflow
@@ -832,6 +846,18 @@ module Workflow
   def real_dependencies(task, orig_jobname, inputs, dependencies)
     real_dependencies = []
     path_deps = {}
+
+    override_dependencies = IndiferentHash.setup({})
+
+    inputs.each do |key,value|
+      if String === key && m = key.match(/(.*)#(.*)/)
+        workflow, task = m.values_at 1, 2
+        workflow = self.to_s if workflow.empty?
+        override_dependencies[workflow] ||= IndiferentHash.setup({})
+        override_dependencies[workflow][task] = value
+      end
+    end
+
     dependencies.each do |dependency|
       _inputs = IndiferentHash.setup(inputs.dup)
       jobname = orig_jobname
@@ -841,20 +867,37 @@ module Workflow
                  when Array
                    workflow, dep_task, options = dependency
 
-                   compute = options[:compute] if options
+                   if override_dependencies[workflow.to_s] && value = override_dependencies[workflow.to_s][dep_task]
+                     d_ = Step === value ? value : Workflow.load_step(value)
+                     d_.task = workflow.tasks[dep_task]
+                     d_.workflow = workflow
+                     d_.overriden = true
+                     d_
+                   else
+
+                     compute = options[:compute] if options
 
-                   all_d = (real_dependencies + real_dependencies.flatten.collect{|d| d.rec_dependencies} ).flatten.compact.uniq
-                   
-                   _inputs = assign_dep_inputs(_inputs, options, all_d, workflow.task_info(dep_task))
-                   jobname = _inputs[:jobname] if _inputs.include? :jobname
+                     all_d = (real_dependencies + real_dependencies.flatten.collect{|d| d.rec_dependencies} ).flatten.compact.uniq
 
-                   job = workflow.job(dep_task, jobname, _inputs)
-                   ComputeDependency.setup(job, compute) if compute
-                   job
+                     _inputs = assign_dep_inputs(_inputs, options, all_d, workflow.task_info(dep_task))
+                     jobname = _inputs[:jobname] if _inputs.include? :jobname
+
+                     job = workflow.job(dep_task, jobname, _inputs)
+                     ComputeDependency.setup(job, compute) if compute
+                     job
+                   end
                  when Step
                    dependency
                  when Symbol
-                   job(dependency, jobname, _inputs)
+                   if override_dependencies[self.to_s] && value = override_dependencies[self.to_s][dependency]
+                     d_ = Step === value ? value : Workflow.load_step(value)
+                     d_.task = self.tasks[dependency]
+                     d_.workflow = self
+                     d_.overriden = true
+                     d_
+                   else
+                     job(dependency, jobname, _inputs)
+                   end
                  when Proc
                    if DependencyBlock === dependency
                      orig_dep = dependency.dependency 
@@ -874,10 +917,18 @@ module Workflow
                        if Hash === d
                          d[:workflow] ||= wf 
                          d[:task] ||= task_name
-                         task_info = d[:workflow].task_info(d[:task])
-
-                         inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
-                         d = d[:workflow].job(d[:task], d[:jobname], inputs) 
+                         if override_dependencies[d[:workflow].to_s] && value = override_dependencies[d[:workflow].to_s][d[:task]]
+                           d = (Step === value ? value : Workflow.load_step(value))
+                           d.task = d[:workflow].tasks[d[:task]]
+                           d.workflow = self
+                           d.overriden = true
+                           d
+                         else
+                           task_info = d[:workflow].task_info(d[:task])
+
+                           inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info) 
+                           d = d[:workflow].job(d[:task], d[:jobname], inputs) 
+                         end
                        end
                        ComputeDependency.setup(d, compute) if compute
                        new_ << d
@@ -888,9 +939,16 @@ module Workflow
                      dep = dependency.call jobname, _inputs, real_dependencies
                      if Hash === dep
                        dep[:workflow] ||= wf  || self
-                       task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
-                       inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
-                       dep = dep[:workflow].job(dep[:task], dep[:jobname], inputs)
+                       if override_dependencies[d[:workflow].to_s] && value = override_dependencies[d[:workflow].to_s][d[:task]]
+                         dep = (Step === value ? value : Workflow.load_step(value))
+                         dep.task = d[:workflow].tasks[d[:task]]
+                         dep.workflow = self
+                         dep.overriden = true
+                       else
+                         task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
+                         inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
+                         dep = dep[:workflow].job(dep[:task], dep[:jobname], inputs)
+                       end
                      end
                    end
 
@@ -915,7 +973,7 @@ module Workflow
                          key_obj = {:inputs => clean_inputs, :dependencies => dependencies}
                          key_str = Misc.obj2str(key_obj)
                          hash_str = Misc.digest(key_str)
-                         Log.debug "Hash for '#{[taskname, jobname] * "/"}' #{hash_str} for #{key_str}"
+                         #Log.debug "Hash for '#{[taskname, jobname] * "/"}' #{hash_str} for #{key_str}"
                          jobname + '_' << hash_str
                        when :inputs
                          all_inputs = {}

data/lib/rbbt/workflow/step.rb

@@ -6,6 +6,7 @@ require 'rbbt/workflow/accessor'
 
 class Step
   attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
+  attr_accessor :task_name, :overriden
   attr_accessor :pid
   attr_accessor :exec
   attr_accessor :result, :mutex, :seen
@@ -28,6 +29,14 @@ class Step
                     end
   end
 
+  def overriden
+    if @overriden.nil? 
+      dependencies.select{|dep| dep.overriden }.any?
+    else
+      @overriden
+    end
+  end
+
   def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
     path = Path.setup(Misc.sanitize_filename(path)) if String === path
     path = path.call if Proc === path
@@ -100,8 +109,13 @@ class Step
   end
 
   def task_name
-    return @path.split("/")[-2] if @task.nil?
-    @task.name
+    @task_name ||= begin
+                     if @task.nil?
+                        @path.split("/")[-2]
+                     else
+                       @task.name
+                     end
+                   end
   end
 
   def path
@@ -259,7 +273,7 @@ class Step
     pid_file = Step.pid_file path
     files_dir = Step.files_dir path
 
-    if Open.exists?(path) or Open.exists?(pid_file) or Open.exists?(info_file)
+    if Open.exists?(path) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir)
 
       @result = nil
       @pid = nil
@@ -291,7 +305,7 @@ class Step
     self
   end
 
-  def rec_dependencies
+  def rec_dependencies(need_run = false)
 
     # A step result with no info_file means that it was manually
     # placed. In that case, do not consider its dependencies
@@ -301,7 +315,9 @@ class Step
 
     new_dependencies = []
     dependencies.each{|step| 
-      r = step.rec_dependencies
+      next if self.done? && need_run && Open.exists?(step.info_file) 
+
+      r = step.rec_dependencies(need_run)
       new_dependencies.concat r
       new_dependencies << step
     }

data/lib/rbbt/workflow/step/dependencies.rb

@@ -290,7 +290,7 @@ class Step
   def run_dependencies
     dep_step = {}
 
-    rec_dependencies = self.rec_dependencies
+    rec_dependencies = self.rec_dependencies(true)
 
     return if rec_dependencies.empty?
 

data/lib/rbbt/workflow/step/run.rb

@@ -108,6 +108,7 @@ class Step
     rec_dependencies.
       select{|dependency| ! (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency) }.
       select{|dependency| ! Open.remote?(dependency.path) }.
+      select{|dependency| Open.exists?(dependency.info_file) }.
       select{|dependency| ! dependency.error? }
   end
 
@@ -126,9 +127,15 @@ class Step
     outdated_dep  = []
     canfail_paths = self.canfail_paths
     checks.each do |dep| 
-      if dep.done? && self.done? && (File.mtime(dep.path) > File.mtime(self.path))
-        outdated_time << dep
+      next unless Open.exists?(dep.info_file)
+
+      begin
+        if dep.done? && self.done? && Open.exists?(dep.path) && Open.exists?(self.path) && (File.mtime(dep.path) > File.mtime(self.path))
+          outdated_time << dep
+        end
+      rescue
       end
+
       if (! dep.done? && ! canfail_paths.include?(dep.path)) || ! dep.updated?
         outdated_dep << dep
       end

data/lib/rbbt/workflow/usage.rb

@@ -32,7 +32,12 @@ module Task
       seen = []
       task_inputs = dep_inputs deps, workflow
       task_inputs.each do |task,new_inputs|
-        task.inputs.zip(task.input_types.values_at(*task.inputs)).select{|i,t| t.to_sym == :select and task.input_options[i][:select_options] }.each{|i,t| selects << [i, task.input_options[i][:select_options]]  }
+        new_inputs.zip(task.input_types.values_at(*new_inputs)).select do |i,t| 
+          t.to_sym == :select and task.input_options[i][:select_options] 
+        end.each do |i,t| 
+          selects << [i, task.input_options[i][:select_options]] 
+        end
+
         if task.workflow and task.workflow != workflow
           puts "  #{Log.color :yellow, ["[#{task.workflow.to_s}]", task.name.to_s] *" "}:"
         else

data/share/install/software/lib/install_helpers

@@ -93,10 +93,13 @@ get_git(){
     cd $OPT_SCM_DIR
     if [ -d "$name" ]; then
         cd "$name" 
-        git stash
-        git pull origin master
-        git rebase master
-        git stash apply
+
+        if [ ! "$NOWEB" == "true" ]; then
+          git stash
+          git pull origin master
+          git rebase master
+          git stash apply
+        fi
     else
         git clone "$url" "$name"
     fi
@@ -211,11 +214,23 @@ setup(){
         local old_pwd="`expand_path $(pwd)`"
         cd "$OPT_DIR/bin" 
 
-        for exe in `ls "$pkg_dir/bin/"`;do 
+        for exe in ` find "$pkg_dir/bin/" -maxdepth 1 -type f -executable`; do
+            exe=$(basename $exe)
             rm -f ./$exe
             ln -s "$pkg_dir/bin/$exe" . 2>/dev/null
         done
 
+        cd "$old_pwd"
+    else
+        local old_pwd="`expand_path $(pwd)`"
+        cd "$OPT_DIR/bin" 
+
+        for exe in ` find "$pkg_dir/" -maxdepth 1 -type f -executable`; do
+            exe=$(basename $exe)
+            rm -f ./$exe
+            ln -s "$pkg_dir/$exe" . 2>/dev/null
+        done
+
         cd "$old_pwd"
     fi
     
@@ -236,6 +251,27 @@ install_src(){
     build "$name"  "$extra"
 }
 
+custom_build(){
+    local name="$1"
+    shift;
+    local cmd="$@"
+
+    echo "Building $name"
+
+    local old_pwd="`expand_path $(pwd)`"
+    cd "`build_dir`"
+    echo `pwd`
+    ls
+
+    eval $cmd
+
+    move_opt "$name"
+    setup "$name"
+    clean_build
+    
+    cd "$old_pwd"
+}
+
 install_git(){
     local name="$1"
     local url="$2"

data/share/rbbt_commands/system/clean

@@ -14,6 +14,7 @@ Clean orphaned files
 $ rbbt system clean <workflow> <task>
 
 -a--all Apply to all jobs, not only uncompleted
+-o--older* Clean jobs not access in some time
 -f--force Remove locks and files regardless of been active
 -q--quick Quick check
 -d--dirty Clean dirty jobs
@@ -29,6 +30,9 @@ task = task.split "," if task
 all = options.delete :all
 force = options.delete :force
 dirty = options.delete :dirty
+time = options.delete :older
+
+time = Misc.timespan time, 'd' if time
 
 puts Log.color(:magenta, "# System clean")
 
@@ -103,8 +107,17 @@ TSV.traverse jobs, :_bar => "Checking job status" do |file,i|
     status = status.to_s
   end
 
-  next unless (force and status !~ /done/) or status == 'dirty' or status =~ /\bnopid/ or status =~ /\berror$/ or status =~ /\bmissing$/ or status =~ /\baborted$/ or status =~ /\bdead$/ or status =~ /\bsync$/ or status == "" 
-  puts "  Removing #{ file } - #{status}" 
-  Step.clean(file) 
+  if time and File.exists?(file)
+    old = Time.now - File.atime(file)
+    status = 'old'
+  end
+
+  if (force and status !~ /done/) or 
+    status =~ /\b(old|dirty|nopid|error|missing|aborted|dead|sync)$/ or
+    status == "" 
+
+    puts "  Removing #{ file } - #{status}" 
+    Step.clean(file) 
+  end
 end
 

data/share/rbbt_commands/system/status

@@ -52,8 +52,10 @@ def status_msg(status)
             :red
           when :streaming, :started
             :cyan
-          when :done, :noinfo
+          when :done
             :green
+          when :noinfo
+            :blue
           when :dependencies, :waiting, :setyp
             :yellow
           else

data/share/rbbt_commands/tsv/get

@@ -17,6 +17,7 @@ Use - to read from STDIN
 -k--key_field* Use this field as key
 -h--help Print this help
 -l--lines Separate in lines
+-t--type* Type of tsv
 EOF
 
 rbbt_usage and exit 0 if options[:help]
@@ -44,11 +45,13 @@ fields = options[:fields]
 key_field = options[:key_field]
 fields = fields.split(/[,|]/, -1) unless fields.nil?
 
-parser = TSV::Parser.new tsv, :key_field => key_field,  :fields => fields
+parser = TSV::Parser.new tsv, :key_field => key_field,  :fields => fields, :type => options[:type]
 fields ||= parser.fields
 
 TSV.traverse(parser) do |k,v|
   next unless k.include? key
+  k = k.first if Array === k
+  puts Log.color(:blue, "Key: #{ k }")
   if fields.length == 1
     if options[:lines]
       puts (Array === v ? v.flatten*"\n" : v.to_s ) 

data/share/rbbt_commands/workflow/info

@@ -37,8 +37,10 @@ def status_msg(status)
             :red
           when :streaming, :started
             :cyan
-          when :done, :noinfo
+          when :done
             :green
+          when :noinfo
+            :blue
           when :dependencies, :waiting, :setyp
             :yellow
           else

data/share/rbbt_commands/workflow/prov

@@ -45,8 +45,10 @@ def status_msg(status)
             :red
           when :streaming, :started
             :cyan
-          when :done, :noinfo
+          when :done
             :green
+          when :noinfo
+            :blue
           when :dependencies, :waiting, :setyp
             :yellow
           else
@@ -60,16 +62,18 @@ def status_msg(status)
 end
 
 def report_msg(status, name, path, info = nil)
-
   parts = path.sub(/\{.*/,'').sub(/#{Regexp.quote(name)}$/,'').split "/"
 
   task = Log.color(:yellow, parts.pop)
   workflow = Log.color(:magenta, parts.pop)
+  if status.to_s == 'noinfo' and parts.last != 'jobs'
+    task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:blue, "file"), Log.color(:magenta, "-")
+  end
 
   str = if not Open.remote?(path) and (File.exists?(path) and $main_mtime and ($main_mtime - File.mtime(path)) < 0)
-    status_msg(status.to_s) << " " << [workflow, task, path] * " " << " (#{Log.color(:red, "Mtime out of sync") })"
+    status_msg(status.to_s) << " " << [workflow, task, path].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
   else
-    status_msg(status.to_s) << " " << [workflow, task, path] * " " 
+    status_msg(status.to_s) << " " << [workflow, task, path].compact * " " 
   end
 
   if $inputs and $inputs.any? 
@@ -97,8 +101,9 @@ def report_msg(status, name, path, info = nil)
   str << "\n"
 end
 
-def report(step, offset = 0)
-  info = step.info || {}
+def report(step, offset = 0, task = nil)
+  info = step.info  || {}
+  info[:task_name] = task
   path  = step.path
   status = info[:status] || :missing
   status = "remote" if Open.remote?(path)
@@ -110,9 +115,9 @@ def report(step, offset = 0)
     new = ! $seen.include?(path)
     dep = get_step path
     if new
-      str << report(dep, offset + 1)
+      str << report(dep, offset + 1, task)
     else
-      str << Log.color(:blue, Log.uncolor(report(dep, offset+1)))
+      str << Log.color(:blue, Log.uncolor(report(dep, offset+1, task)))
     end
   end if info[:dependencies]
   str

data/share/rbbt_commands/workflow/task

@@ -202,6 +202,7 @@ the job dependencies recursively.
 -W--workflows* Load a list of workflows
 -R--requires* Require a list of files
 -rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
+-od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
 EOF
 
 workflow = ARGV.shift
@@ -216,6 +217,7 @@ detach = !!options.delete(:detach)
 do_exec = !!options.delete(:exec)
 clean = !!options.delete(:clean)
 clean_task = options.delete(:clean_task)
+override_deps = options.delete(:override_deps)
 recursive_clean = !!options.delete(:recursive_clean)
 out = options.include?(:output) ? File.open(options[:output], 'wb') : STDOUT
 
@@ -294,6 +296,13 @@ saved_job_options = job_options
 
 workflow.workdir = Path.setup(File.expand_path(options.delete(:workdir))) if options[:workdir]
 
+if override_deps
+  override_deps.split($array_separator).each do |part|
+    t_, value = part.split("=")
+    job_options.merge!( t_ => value)
+  end
+end
+
 #- get job
 
 job = workflow.job(task.name, name, job_options)
@@ -306,7 +315,9 @@ end
 
 if clean_task
   job.rec_dependencies.each do |dep|
-    dep.clean if dep.task_name.to_s == clean_task.to_s
+    next unless dep.task_name.to_s == clean_task.to_s
+    dep.clean 
+    dep.set_info :status, :cleaned
   end
 end
 

data/test/rbbt/resource/test_path.rb

@@ -17,6 +17,16 @@ class TestTSV < Test::Unit::TestCase
     end
   end
 
+  def test_find_current
+    TmpFile.with_file do |dir|
+      Misc.in_dir dir do
+        CMD.cmd('touch foo')
+        p = Path.setup('foo')
+        assert_equal File.join(dir, 'foo'), p.find(:current)
+      end
+    end
+  end
+
   def test_prev
     path = Path.setup "/tmp"
     assert_equal "/tmp/bar/foo", path.foo("bar")
@@ -31,9 +41,9 @@ class TestTSV < Test::Unit::TestCase
     assert_equal path.find, path.doc_file.find(:lib).source_for_doc_file
     assert_equal path.find, path.doc_file.source_for_doc_file.find
 
-    assert_equal "doc/lib/rbbt/resource.doc", path.doc_file.set_extension('doc')
-    assert_equal "lib/rbbt/resource.rb", path.doc_file.set_extension('doc').source_for_doc_file
+    assert_equal "doc/lib/rbbt/resource.rb.doc", path.doc_file.set_extension('doc')
+    #assert_equal "lib/rbbt/resource.rb", path.doc_file.set_extension('doc').source_for_doc_file
 
-    assert_equal "doc/lib/rbbt/resource.doc", path.doc_file.set_extension('doc')
+    assert_equal "doc/lib/rbbt/resource.rb.doc", path.doc_file.set_extension('doc')
   end
 end

data/test/rbbt/test_resource.rb

@@ -33,7 +33,8 @@ class TestTSV < Test::Unit::TestCase
     assert TSV === TestResource.tmp.test.test_tsv.tsv
   end
 
-  def test_rake
+  def __test_rake
+    iii TestResource.tmp.test.work.footest.foo.find
     TestResource.tmp.test.work.footest.foo.read == "TEST"
     assert TestResource.tmp.test.work.footest.foo.read == "TEST"
   end

data/test/rbbt/test_tsv.rb

@@ -98,7 +98,7 @@ row2    A    B    Id3
     end
   end
 
-  def test_tsv_field_selection
+  def test_tsv_field_selection2
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2

data/test/rbbt/test_workflow.rb

@@ -107,6 +107,23 @@ for this dependency
   task :send_input_dep_to_reverse => :text do
     TSV.get_stream step(:reverse_input_text)
   end
+
+  input :i, :string, "Input", "A"
+  task :t1 => :string do |i|
+    i
+  end
+
+  input :i, :string, "Input", "B"
+  task :t2 => :string do |i|
+    i
+  end
+
+  dep :t1, :i => "C"
+  dep :t2
+  task :t3 => :string do |i|
+    step(:t1).load + step(:t2).load
+  end
+
 end
 
 TestWF.workdir = Rbbt.tmp.test.workflow
@@ -198,6 +215,12 @@ class TestWorkflow < Test::Unit::TestCase
     assert_equal "V:AA", job.run
   end
 
+  def test_override_dep
+    TmpFile.with_file("OTHER") do |file|
+      assert TestWF.job(:repeat2, nil, :number => 3, "TestWF#str" => file).run.include? "OTHER"
+    end
+  end
+
   def __test_stream
     io = TestWF.job(:stream).run(:stream)
     Misc.consume_stream(TSV.get_stream(io), false, STDOUT)
@@ -228,4 +251,9 @@ class TestWorkflow < Test::Unit::TestCase
     assert TestWF.rec_input_use(:double_dep)[:times][TestWF].include?(:repeat)
   end
 
+  def test_shared_inputs
+    assert_equal "CB", TestWF.job(:t3).run
+    assert_equal "CB", TestWF.job(:t3).run
+  end
+
 end

data/test/rbbt/util/test_misc.rb

@@ -602,4 +602,12 @@ EOF
     assert_equal 1, hash[:A]
     assert_equal 3, hash[:C]
   end
+  
+  def test_time_stamp
+    assert_equal 10, Misc.timespan('10')
+    assert_equal 10, Misc.timespan('10s')
+    assert_equal 60, Misc.timespan('1m')
+    assert_equal 60*60*24, Misc.timespan('1d')
+    assert_equal 60*60*24, Misc.timespan('1d')
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.22.4
+  version: 5.22.5
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-07-05 00:00:00.000000000 Z
+date: 2018-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake