rbbt-util 5.27.1 → 5.27.6

data/lib/rbbt/resource.rb

@@ -170,10 +170,12 @@ module Resource
     if type and not File.exist?(final_path) or force
       Log.medium "Producing: #{ final_path }"
       lock_filename = Persist.persistence_path(final_path, {:dir => Resource.lock_dir})
+
       Misc.lock lock_filename do
         FileUtils.rm_rf final_path if force and File.exist? final_path
-        if not File.exist?(final_path) or force
-          (remote_server and get_from_server(path, final_path)) or
+
+        if ! File.exist?(final_path) || force 
+
           begin
             case type
             when :string
@@ -291,7 +293,7 @@ url='#{url}'
           rescue
             FileUtils.rm_rf final_path if File.exist? final_path
             raise $!
-          end
+          end unless (remote_server && get_from_server(path, final_path))
         end
       end
     end
@@ -304,11 +306,12 @@ url='#{url}'
     resource ||= Rbbt
     (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys).uniq.each do |name|
       pattern = resource.search_paths[name]
-      next if patterns.nil?
+      next if pattern.nil?
+      pattern = pattern.sub('{PWD}', Dir.pwd)
       if String ===  pattern and pattern.include?('{')
         regexp = "^" + pattern.gsub(/{([^}]+)}/,'(?<\1>[^/]+)') + "(?:/(?<REST>.*))?/?$"
         if m = path.match(regexp) 
-          if m["PKGDIR"] == resource.pkgdir
+          if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == resource.pkgdir
             return self[m["TOPLEVEL"]][m["SUBPATH"]][m["REST"]]
           end
         end

data/lib/rbbt/resource/path.rb

@@ -199,7 +199,7 @@ module Path
                          next if res
                          next unless paths.include? w
                          path = find(w, caller_lib, paths)
-                         res = path if File.exist? path
+                         res = path if File.exist?(path)
                        end if res.nil?
 
                        (paths.keys - STANDARD_SEARCH - search_order).each do |w|
@@ -241,8 +241,8 @@ module Path
                          sub('{REMOVE}/', '').
                          sub('{REMOVE}', '')
 
-                       path = path + '.gz' if File.exist? path + '.gz'
-                       path = path + '.bgz' if File.exist? path + '.bgz'
+                       path = path + '.gz' if File.exist?(path + '.gz')
+                       path = path + '.bgz' if File.exist?(path + '.bgz')
 
                        self.annotate path
 

data/lib/rbbt/tsv/accessor.rb

@@ -243,7 +243,7 @@ module TSV
   end
 
   def []=(key, value, clean = false)
-    return super(key, value) if clean or value.nil? or TSV::CleanSerializer == self.serializer_module 
+    return super(key, value) if clean || value.nil? || TSV::CleanSerializer == self.serializer_module 
     super(key, @serializer_module.dump(value))
   end
 
@@ -366,11 +366,11 @@ module TSV
     keys.length
   end
 
-  def values_at(*keys)
-    keys.collect do |key|
-      self[key]
-    end
-  end
+  #def _values_at(*keys)
+  #  keys.collect do |key|
+  #    self[key]
+  #  end
+  #end
 
   def chunked_values_at(keys, max = 5000)
     Misc.ordered_divide(keys, max).inject([]) do |acc,c|

data/lib/rbbt/tsv/change_id.rb

@@ -10,7 +10,8 @@ module TSV
 
     identifiers = Organism.identifiers(tsv.namespace) if identifiers.nil? and tsv.namespace
 
-    if not tsv.fields.include? format
+
+    if ! tsv.fields.include?(format)
       new = {}
       tsv.each do |k,v|
         if v === String or v === Array
@@ -32,6 +33,7 @@ module TSV
         tsv = tsv.attach identifiers, :fields => [format], :persist_input => true
       end
 
+
       tsv = tsv.reorder(format, tsv.fields[0..-2])
 
       tsv = tsv.to_flat  if orig_type == :flat

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -374,7 +374,7 @@ module TSV
       when (defined? Step and Step)
 
         obj.clean if obj.aborted? or obj.recoverable_error?
-        obj.run(true) unless obj.done? or obj.started?
+        obj.run(true) unless obj.done? || obj.started? || obj.result
 
         stream = obj.get_stream
 

data/lib/rbbt/tsv/util.rb

@@ -122,6 +122,7 @@ module TSV
         end
       else
         file.grace
+
         stream = file.get_stream
         if stream && ! stream.closed?
           stream

data/lib/rbbt/util/misc/exceptions.rb

@@ -15,6 +15,14 @@ end
 class Aborted < StandardError; end
 
 class TryAgain < StandardError; end
+
+class TryThis < StandardError
+  attr_accessor :payload
+  def initialize(payload = nil)
+    @payload = payload
+  end
+end
+
 class SemaphoreInterrupted < TryAgain; end
 class LockInterrupted < TryAgain; end
 

data/lib/rbbt/workflow.rb

@@ -607,7 +607,7 @@ module Workflow
   end
 
   def self.load_step(path)
-    Path.setup(path) unless Path === path
+    path = Path.setup(path.dup) unless Path === path
     path = path.find
 
     begin

data/lib/rbbt/workflow/accessor.rb

@@ -35,6 +35,7 @@ module Workflow
                            input_use = rec_input_use(name)
                            input_defaults = rec_input_defaults(name)
                            input_options = rec_input_options(name)
+                           extension = task.extension
                            export = case
                                     when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
                                       :synchronous
@@ -60,7 +61,8 @@ module Workflow
                              :input_use => input_use,
                              :result_type => result_type,
                              :result_description => result_description,
-                             :dependencies => dependencies
+                             :dependencies => dependencies,
+                             :extension => extension
                            }
                          end
   end

data/lib/rbbt/workflow/definition.rb

@@ -18,6 +18,7 @@ module Workflow
     :description        => "",
     :result_type        => nil,
     :result_description => "",
+    :resumable          => false,
     :extension          => nil)
 
 
@@ -33,6 +34,10 @@ module Workflow
     @extension = extension
   end
 
+  def resumable
+    @resumable = true
+  end
+
   def returns(description)
     @result_description = description
   end
@@ -118,6 +123,7 @@ module Workflow
       :input_descriptions => consume_input_descriptions,
       :required_inputs    => consume_required_inputs,
       :extension          => consume_extension,
+      :resumable          => consume_resumable,
       :input_options      => consume_input_options
     }
 

data/lib/rbbt/workflow/remote_workflow/driver/rest.rb

@@ -137,13 +137,15 @@ class RemoteWorkflow
 
         post_thread = Thread.new(Thread.current) do |parent|
           bl = lambda do |rok|
-            if Net::HTTPOK === rok
+            if Net::HTTPOK === rok 
               _url = rok["RBBT-STREAMING-JOB-URL"]
               @url = File.join(task_url, File.basename(_url)) if _url
               rok.read_body do |c,_a, _b|
                 sin.write c
               end
               sin.close
+            elsif Net::HTTPSeeOther === rok
+              raise TryThis.new(rok)
             else
               err = StringIO.new
               rok.read_body do |c,_a, _b|
@@ -156,7 +158,7 @@ class RemoteWorkflow
                        err.rewind
                        err.read
                      end
-              ne = @adaptor.parse_exception text
+              ne = RemoteWorkflow.parse_exception text
               case ne
               when String
                 parent.raise e.class, ne
@@ -173,7 +175,11 @@ class RemoteWorkflow
           end
 
           Log.debug{ "RestClient execute: #{ task_url } - #{Misc.fingerprint task_params}" }
-          RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
+          begin
+            RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
+          rescue TryThis
+            RestClient::Request.execute(:method => :get, :url => $!.payload.header[:location], :block_response => bl)
+          end
         end
 
         # It seems like now response body are now decoded by Net::HTTP after 2.1

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -25,8 +25,13 @@ class RemoteStep < Step
   end
 
   def cache_file
-    digest = Misc.obj2digest([base_url, task, base_name, inputs])
-    Rbbt.var.cache.REST[[task, clean_name, digest] * "."].find
+    begin
+      digest = Misc.obj2digest([base_url, task, base_name, inputs])
+      Rbbt.var.cache.REST[[task, clean_name, digest].compact * "."].find
+    rescue
+      Log.exception $!
+      raise $!
+    end
   end
 
   def cache_files
@@ -62,7 +67,6 @@ class RemoteStep < Step
     no_load ? Misc.add_GET_param(path, "_format", "raw") : @result
   end
 
-
   def self.get_streams(inputs, stream_input = nil)
     new_inputs = {}
     inputs.each do |k,v|
@@ -240,6 +244,7 @@ class RemoteStep < Step
     return true if cache_files.any?
     init_job unless @url
     Log.debug{ "Joining RemoteStep: #{path}" }
+
     if IO === @result
       res = @result
       @result = nil
@@ -253,6 +258,7 @@ class RemoteStep < Step
       sleep 1 unless self.done? || self.aborted? || self.error?
       while not (self.done? || self.aborted? || self.error?)
         sleep 3
+        iif [self.done?, self.status, self.info]
       end
     end
 

data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb

@@ -129,7 +129,7 @@ class RemoteStep
       end
     end
 
-    def _run_job(cache_type = :async)
+    def _run_job(cache_type = :asynchronous)
       get_streams
 
       task_url = URI.encode(File.join(base_url, task.to_s))
@@ -142,7 +142,13 @@ class RemoteStep
       else
         @adaptor.execute_job(base_url, task, task_params, cache_type)
       end
+    end
+
 
+    def produce(*args)
+      @started = true
+      _run_job
     end
+
   end
 end

data/lib/rbbt/workflow/step/accessor.rb

@@ -612,6 +612,10 @@ class Step
     provenance
   end
 
+  def resumable?
+    task && task.resumable
+  end
+
   def config(key, *tokens)
     options = tokens.pop if Hash === tokens.last
     options ||= {}

data/lib/rbbt/workflow/step/dependencies.rb

@@ -92,7 +92,7 @@ class Step
           (job.done? && job.dirty?)  || (job.error? && job.dirty?) ||
           (!(job.noinfo? || job.done? || job.error? || job.aborted? || job.running?))
 
-        job.clean 
+        job.clean unless job.resumable? && (job.updated? && ! job.dirty?)
         job.set_info :status, :cleaned
       end
 
@@ -144,14 +144,18 @@ class Step
 
       dependency.status_lock.synchronize do
         if dependency.aborted? || (dependency.error? && dependency.recoverable_error? && ! canfail_paths.include?(dependency.path) && ! already_failed.include?(dependency.path)) || (!Open.remote?(dependency.path) && dependency.missing?)
-          Log.warn "Cleaning dep. on exec #{Log.color :blue, dependency.path} (missing: #{dependency.missing?}; error #{dependency.error?})"
-          dependency.clean
-          already_failed << dependency.path
-          raise TryAgain
+          if dependency.resumable?
+            dependency.status = :resume
+          else
+            Log.warn "Cleaning dep. on exec #{Log.color :blue, dependency.path} (missing: #{dependency.missing?}; error #{dependency.error?})"
+            dependency.clean
+            already_failed << dependency.path
+            raise TryAgain
+          end
         end
       end
 
-      if ! (dependency.started? || dependency.error?)
+      if dependency.status == :resume || ! (dependency.started? || dependency.error?)
         log_dependency_exec(dependency, :starting)
         dependency.run(true)
         raise TryAgain

data/lib/rbbt/workflow/step/run.rb

@@ -108,7 +108,7 @@ class Step
       @result = self._exec
       @result = @result.stream if TSV::Dumper === @result
     end
-    (no_load or ENV["RBBT_NO_STREAM"]) ? @result : prepare_result(@result, @task.result_description)
+    (no_load || ENV["RBBT_NO_STREAM"]) ? @result : prepare_result(@result, @task.result_description)
   end
 
   def updatable?
@@ -610,7 +610,7 @@ class Step
   end
 
   def _clean_finished
-    if Open.exists? path and not status == :done
+    if Open.exists?(path) && status != :done
       Log.warn "Aborted job had finished. Removing result -- #{ path }"
       begin
         Open.rm path

data/lib/rbbt/workflow/task.rb

@@ -2,7 +2,7 @@ require 'rbbt/util/misc'
 require 'rbbt/persist'
 
 module Task
-  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :required_inputs, :description, :name, :result_description, :extension, :workflow
+  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :required_inputs, :description, :name, :result_description, :extension, :workflow, :resumable
 
   def self.setup(options = {}, &block)
     block.extend Task

data/lib/rbbt/workflow/usage.rb

@@ -71,7 +71,7 @@ module Workflow
     @dep_tree ||= {}
     @dep_tree[name] ||= begin
                         dep_tree = {}
-                        self.task_dependencies[name].reverse.each do |dep|
+                        self.task_dependencies[name.to_sym].reverse.each do |dep|
                           dep = dep.first if Array === dep && dep.length == 1
                           dep = dep.dependency if DependencyBlock === dep
 
@@ -88,23 +88,38 @@ module Workflow
                           key = [workflow, task]
 
                           dep_tree[key] = workflow.dep_tree(task)
-                        end
+                        end if self.task_dependencies[name.to_sym]
                         dep_tree
                       end
   end
 
+  def _prov_tasks(tree)
+    tasks = [] 
+    heap = [tree]
+    while heap.any?
+      t = heap.pop
+      t.each do |k,v|
+        tasks << k
+        heap << v
+      end
+    end
+    tasks
+  end
+
   def prov_string(tree)
     description = ""
 
     last = nil
     seen = Set.new
-    tree.collect.to_a.flatten.select{|e| Symbol === e }.each do |task_name|
 
-      child = last && last.include?(task_name)
-      first = last.nil?
-      last = dep_tree(task_name).collect.to_a.flatten.select{|e| Symbol === e}
+    tasks = _prov_tasks(tree)
+    tasks.each do |workflow,task_name|
+
+      next if seen.include?([workflow,task_name])
 
-      next if seen.include?(task_name)
+      child = last && last.include?([workflow, task_name])
+      first = last.nil?
+      last = _prov_tasks(workflow.dep_tree(task_name))
 
       if child
         description << "->" << task_name.to_s
@@ -113,6 +128,8 @@ module Workflow
       else
         description << ";" << task_name.to_s
       end
+      
+      seen << [workflow, task_name]
     end
     description
   end
@@ -177,7 +194,7 @@ module Workflow
         description = task.description || ""
         description = description.split("\n\n").first
 
-        next if abridge and ! final.include?(name)
+        next if abridge && ! final.include?(name)
         puts Misc.format_definition_list_item(name.to_s, description, Log.terminal_width, 20, :yellow)
 
         prov_string = prov_string(dep_tree(name))

data/lib/rbbt/workflow/util/archive.rb

@@ -125,6 +125,7 @@ class Step
   def self.migrate(path, search_path, options = {})
     resource=Rbbt
 
+    orig_path = path
     other_rsync_args = options[:rsync]
 
     recursive = options[:recursive]
@@ -154,6 +155,7 @@ puts files * "\n"
             else
               if File.exists?(path)
                 path = resource.identify(path)
+                raise "Resource #{resource} could not identify #{orig_path}" if path.nil?
               else
                 path = Path.setup(path)
               end
@@ -178,7 +180,7 @@ puts resource[path].find(search_path)
     subpath_files = {}
     paths.sort.each do |path|
       parts = path.split("/")
-      subpath = parts[0..-4] * "/"
+      subpath = parts[0..-4] * "/" + "/"
 
       if subpath_files.keys.any? && subpath.start_with?(subpath_files.keys.last)
         subpath = subpath_files.keys.last
@@ -190,6 +192,7 @@ puts resource[path].find(search_path)
       subpath_files[subpath] << source
     end
 
+    synced_files = []
     subpath_files.each do |subpath, files|
       if options[:target]
         CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
@@ -204,11 +207,15 @@ puts resource[path].find(search_path)
       end
       target = [options[:target], target] * ":" if options[:target]
 
+      next if File.exists?(source) && File.exists?(target) && File.expand_path(source) == File.expand_path(target)
+
       files_and_dirs = Set.new( files )
       files.each do |file|
+        synced_files << File.join(subpath, file)
+
         parts = file.split("/")[0..-2].reject{|p| p.empty?}
         while parts.any?
-          files_and_dirs << parts * "/"
+          files_and_dirs <<  parts * "/"
           parts.pop
         end
       end
@@ -218,8 +225,7 @@ puts resource[path].find(search_path)
 
         cmd = "rsync #{MAIN_RSYNC_ARGS} --progress #{test_str} --files-from='#{tmp_include_file}' #{source}/ #{target}/ #{other_rsync_args}"
 
-        cmd << " && rm -Rf #{source}" if options[:delete]
-
+        #cmd << " && rm -Rf #{source}" if options[:delete]
         if options[:print]
           ppp Open.read(tmp_include_file)
           puts cmd 
@@ -228,6 +234,26 @@ puts resource[path].find(search_path)
         end
       end
     end
+
+    if options[:delete] && synced_files.any?
+      puts Log.color :magenta, "About to erase these files:"
+      synced_files.each do |p|
+        puts Log.color :red, p
+      end 
+
+      if options[:non_interactive]
+        response = 'yes'
+      else
+        puts Log.color :magenta, "Type 'yes' if you are sure:"
+        response = STDIN.gets.chomp
+      end
+
+      if response == 'yes'
+        synced_files.each do |p|
+          Open.rm p
+        end
+      end
+    end
   end
 
   def self.purge(path, recursive = false)