rbbt-util 5.27.2 → 5.27.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 258aa01482866ef7b940790ad6f10d077a0cd721b972cbb939412c3691595acc
-  data.tar.gz: 9adb68a2c9ddfa77a3be419dd106e4083e11933dd6653b4d6e0bfa921edab187
+  metadata.gz: 6551d632ae1f2289805a53ceaf9db01d0c0a029ea6c76176aed419e4d6a2a485
+  data.tar.gz: 72562729535554e718451adb87338503ff40a81874a9e0fed6f8eaf4e6aa0edc
 SHA512:
-  metadata.gz: 4df9dcbc8280023faa1e6c2e6a9393a9ae66e4513a95cdeae94256f5fc61f0e6691e114cd1d40a900041e1769988b2a69751f282fc95c0c134acb44cac3ed40c
-  data.tar.gz: b72bd064835be7c5dcfcf180ff87d6e22fced3f8833d9479f607ab64760594c7d7d25d84cd828b33f0a7d72208983b8c89e26d8924e91afc81ba5e5943ef462d
+  metadata.gz: 0d8c3ff2861dd4bbd835e3e41826f5fd6452bdd0b40fbd8378737effdffa6da051ce221e24f4bbacaf952725ccf7da09b533c8c414a7fd10480369ae528a4259
+  data.tar.gz: 1790b8f38f91dfbb582a394d86953156d90351a8d724fb9535a3bc72ebac5fbbfe24e88fe10cf943e2ed91ff059af68cf581bac333f5480096f85cf6e29b8018

data/lib/rbbt/fix_width_table.rb

@@ -4,20 +4,21 @@ class FixWidthTable
   def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
     @filename = filename
 
-    if update or %w(memory stringio).include?(filename.to_s.downcase) or not File.exist?(filename)
+    if update || %w(memory stringio).include?(filename.to_s.downcase) || ! File.exist?(filename)
       Log.debug "FixWidthTable create: #{ filename }"
       @value_size  = value_size
       @range       = range
       @record_size = @value_size + (@range ? 16 : 8)
       @write = true
 
-      if %w(memory stringio).include? filename.to_s.downcase
+      if %w(memory stringio).include?(filename.to_s.downcase)
         @filename = :memory
         @file     = StringIO.new
       else
         FileUtils.rm @filename if File.exist? @filename
         FileUtils.mkdir_p File.dirname(@filename) unless File.exist? @filename
-        @file = File.open(@filename, 'wb')
+        #@file = File.open(@filename, 'wb')
+        @file = File.open(@filename, 'w:ASCII-8BIT')
       end
 
       @file.write [value_size].pack("L")
@@ -25,9 +26,9 @@ class FixWidthTable
 
       @size = 0
     else
-      Log.debug "FixWidthTable up-to-date: #{ filename }"
+      Log.debug "FixWidthTable up-to-date: #{ filename } - (in_memory:#{in_memory})"
       if in_memory
-        @file        = StringIO.new(Open.read(@filename, :mode => 'r:ASCII-8BIT'), 'r')
+        @file        = Open.open(@filename, :mode => 'r:ASCII-ASCII'){|f| StringIO.new f.read}
       else
         @file        = File.open(@filename, 'r:ASCII-8BIT')
       end

data/lib/rbbt/persist/tsv.rb

@@ -148,13 +148,13 @@ module Persist
 
         data.write_and_read do
           yield data
-        end
 
-        FileUtils.mv data.persistence_path, path if File.exist? data.persistence_path and not File.exist? path
-        tsv = CONNECTIONS[path] = CONNECTIONS.delete tmp_path
-        tsv.persistence_path = path
+          FileUtils.mv data.persistence_path, path if File.exist? data.persistence_path and not File.exist? path
+          tsv = CONNECTIONS[path] = CONNECTIONS.delete tmp_path
+          tsv.persistence_path = path
 
-        tsv.fix_io if tsv.respond_to? :fix_io
+          tsv.fix_io if tsv.respond_to? :fix_io
+        end
 
         data
       rescue Exception

data/lib/rbbt/persist/tsv/adapter.rb

@@ -9,25 +9,10 @@ module Persist
       @mutex ||= Mutex.new
     end
 
-    def prefix(key)
-      range(key, 1, key + MAX_CHAR, 1)
-    end
-
-    def get_prefix(key)
-      keys = prefix(key)
-      select(:key => keys)
-    end
-
     def closed?
       @closed
     end
 
-    def close
-      @closed = true
-      super
-      self
-    end
-
     def write?
       @writable
     end
@@ -47,8 +32,10 @@ module Persist
     def close(*args)
       begin
         super(*args)
+        @closed = true
       rescue NoMethodError
       end
+      self
     end
 
     def read(*args)
@@ -58,25 +45,14 @@ module Persist
       end
     end
 
-    def collect
-      res = []
-      each do |key, value|
-        res << if block_given?
-                 yield key, value
-        else
-          [key, value]
-        end
-      end
-      res
-    end
-
     def delete(key)
       out(key)
     end
 
     def lock
-      #return yield if @locked
+      return yield if @locked
       lock_filename = Persist.persistence_path(persistence_path, {:dir => TSV.lock_dir})
+      Log.stack caller if $LOG
       Misc.lock(lock_filename) do
         begin
           @locked = true
@@ -87,6 +63,16 @@ module Persist
       end
     end
 
+    def lock_and_close
+      lock do
+        begin
+          yield
+        ensure
+          close
+        end
+      end
+    end
+
     def write_and_read
       if write?
         begin
@@ -97,7 +83,7 @@ module Persist
       end
 
       lock do
-        write true if closed? or not write?
+        write(true) if closed? || !write?
         begin
           yield
         ensure
@@ -111,12 +97,12 @@ module Persist
         begin
           return yield
         ensure
-          close
+          close unless @locked
         end
       end
 
       lock do
-        write true if closed? || ! write?
+        write(true) if closed? || ! write?
         res = begin
                 yield
               rescue Exception
@@ -130,11 +116,12 @@ module Persist
     end
 
     def read_and_close
-      if read?
+      #return yield if @locked
+      if read? || write?
         begin
           return yield
         ensure
-          close
+          close unless @locked
         end
       end
 
@@ -148,21 +135,85 @@ module Persist
       end
     end
 
-
     def merge!(hash)
       hash.each do |key,values|
         self[key] = values
       end
     end
 
-
     def range(*args)
-      super(*args) #- TSV::ENTRY_KEYS.to_a
+      self.read_and_close do
+        super(*args)
+      end
     end
 
     def include?(*args)
-      read if closed?
-      super(*args) #- TSV::ENTRY_KEYS.to_a
+      self.read_and_close do
+        super(*args) #- TSV::ENTRY_KEYS.to_a
+      end
+    end
+
+    def [](*args)
+      self.read_and_close do
+        super(*args) #- TSV::ENTRY_KEYS.to_a
+      end
+    end
+
+    def []=(*args)
+      self.write_and_close do
+        super(*args) #- TSV::ENTRY_KEYS.to_a
+      end
+    end
+
+    def keys(*args)
+      self.read_and_close do
+        super(*args)
+      end
+    end
+
+
+    def prefix(key)
+      self.read_and_close do
+        range(key, 1, key + MAX_CHAR, 1)
+      end
+    end
+
+    def get_prefix(key)
+      keys = prefix(key)
+      select(:key => keys)
+    end
+
+
+    def size(*args)
+      self.read_and_close do
+        super(*args)
+      end
+    end
+
+    def each(*args, &block)
+      self.read_and_close do
+        super(*args, &block)
+      end
+    end
+
+    def collect
+      res = []
+      each do |key, value|
+        res << if block_given?
+                 yield key, value
+        else
+          [key, value]
+        end
+      end
+      res
+    end
+
+    def values_at(*keys)
+      self.read_and_close do
+        keys.collect do |k|
+          self[k]
+        end
+      end
     end
   end
 end

data/lib/rbbt/persist/tsv/tokyocabinet.rb

@@ -16,7 +16,7 @@ module Persist
       end
 
       dir = File.dirname(File.expand_path(path))
-      Open.mkdir(dir) unless Open.exists?(dir)
+      File.mkdir(dir) unless File.exists?(dir)
 
       tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
       tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
@@ -87,13 +87,15 @@ module Persist
   def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
     write = true unless File.exist? path
 
-    FileUtils.mkdir_p File.dirname(path) unless Open.exist?(File.dirname(path))
+    FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
 
     database = Persist::TCAdapter.open(path, write, tokyocabinet_class)
 
     unless serializer == :clean
       TSV.setup database
-      database.serializer = serializer || database.serializer
+      database.write_and_read do
+        database.serializer = serializer
+      end if serializer && database.serializer != serializer
     end
 
     database

data/lib/rbbt/resource.rb

@@ -170,10 +170,12 @@ module Resource
     if type and not File.exist?(final_path) or force
       Log.medium "Producing: #{ final_path }"
       lock_filename = Persist.persistence_path(final_path, {:dir => Resource.lock_dir})
+
       Misc.lock lock_filename do
         FileUtils.rm_rf final_path if force and File.exist? final_path
-        if not File.exist?(final_path) or force
-          (remote_server and get_from_server(path, final_path)) or
+
+        if ! File.exist?(final_path) || force 
+
           begin
             case type
             when :string
@@ -291,7 +293,7 @@ url='#{url}'
           rescue
             FileUtils.rm_rf final_path if File.exist? final_path
             raise $!
-          end
+          end unless (remote_server && get_from_server(path, final_path))
         end
       end
     end
@@ -304,11 +306,12 @@ url='#{url}'
     resource ||= Rbbt
     (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys).uniq.each do |name|
       pattern = resource.search_paths[name]
-      next if patterns.nil?
+      next if pattern.nil?
+      pattern = pattern.sub('{PWD}', Dir.pwd)
       if String ===  pattern and pattern.include?('{')
         regexp = "^" + pattern.gsub(/{([^}]+)}/,'(?<\1>[^/]+)') + "(?:/(?<REST>.*))?/?$"
         if m = path.match(regexp) 
-          if m["PKGDIR"] == resource.pkgdir
+          if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == resource.pkgdir
             return self[m["TOPLEVEL"]][m["SUBPATH"]][m["REST"]]
           end
         end

data/lib/rbbt/resource/path.rb

@@ -199,7 +199,7 @@ module Path
                          next if res
                          next unless paths.include? w
                          path = find(w, caller_lib, paths)
-                         res = path if File.exist? path
+                         res = path if File.exist?(path)
                        end if res.nil?
 
                        (paths.keys - STANDARD_SEARCH - search_order).each do |w|
@@ -241,8 +241,8 @@ module Path
                          sub('{REMOVE}/', '').
                          sub('{REMOVE}', '')
 
-                       path = path + '.gz' if File.exist? path + '.gz'
-                       path = path + '.bgz' if File.exist? path + '.bgz'
+                       path = path + '.gz' if File.exist?(path + '.gz')
+                       path = path + '.bgz' if File.exist?(path + '.bgz')
 
                        self.annotate path
 

data/lib/rbbt/tsv/accessor.rb

@@ -243,7 +243,7 @@ module TSV
   end
 
   def []=(key, value, clean = false)
-    return super(key, value) if clean or value.nil? or TSV::CleanSerializer == self.serializer_module 
+    return super(key, value) if clean || value.nil? || TSV::CleanSerializer == self.serializer_module 
     super(key, @serializer_module.dump(value))
   end
 
@@ -366,11 +366,11 @@ module TSV
     keys.length
   end
 
-  def values_at(*keys)
-    keys.collect do |key|
-      self[key]
-    end
-  end
+  #def _values_at(*keys)
+  #  keys.collect do |key|
+  #    self[key]
+  #  end
+  #end
 
   def chunked_values_at(keys, max = 5000)
     Misc.ordered_divide(keys, max).inject([]) do |acc,c|

data/lib/rbbt/tsv/change_id.rb

@@ -10,7 +10,8 @@ module TSV
 
     identifiers = Organism.identifiers(tsv.namespace) if identifiers.nil? and tsv.namespace
 
-    if not tsv.fields.include? format
+
+    if ! tsv.fields.include?(format)
       new = {}
       tsv.each do |k,v|
         if v === String or v === Array
@@ -32,6 +33,7 @@ module TSV
         tsv = tsv.attach identifiers, :fields => [format], :persist_input => true
       end
 
+
       tsv = tsv.reorder(format, tsv.fields[0..-2])
 
       tsv = tsv.to_flat  if orig_type == :flat

data/lib/rbbt/tsv/stream.rb

@@ -89,6 +89,7 @@ module TSV
         stream
       end
 
+      all_fields = fields
       key_field = key_fields.compact.first
       if same_fields
         fields = fields.first

data/lib/rbbt/util/misc/exceptions.rb

@@ -15,6 +15,14 @@ end
 class Aborted < StandardError; end
 
 class TryAgain < StandardError; end
+
+class TryThis < StandardError
+  attr_accessor :payload
+  def initialize(payload = nil)
+    @payload = payload
+  end
+end
+
 class SemaphoreInterrupted < TryAgain; end
 class LockInterrupted < TryAgain; end
 

data/lib/rbbt/util/misc/pipes.rb

@@ -18,6 +18,8 @@ module Misc
 
   BLOCK_SIZE=1024 * 8
 
+  SKIP_TAG="[SKIP TAG]"
+
   PIPE_MUTEX = Mutex.new
 
   OPEN_PIPE_IN = []
@@ -29,7 +31,7 @@ module Misc
 
       [sout, sin]
     end
-    Log.debug{"Creating pipe #{[res.last.inspect,res.first.inspect] * " => "}"}
+    Log.debug{"Creating pipe #{[res.last.inspect, res.first.inspect] * " => "}"}
     res
   end
 
@@ -255,6 +257,11 @@ module Misc
     end
     tee1, *rest = Misc.tee_stream stream_dup, num + 1
     stream.reopen(tee1)
+
+    #ToDo: I can't explain why the @threads variable appears with the value of
+    # @filename
+    stream.instance_variable_set(:@threads, nil) if stream.instance_variables.include?(:@threads)
+
     tee1.annotate(stream)
     rest
   end
@@ -537,18 +544,29 @@ module Misc
     end
   end
 
+  def self.buffer_stream(stream)
+    sout, sin = Misc.pipe
+    Misc.consume_stream(stream, true, sin)
+    sout
+  end
+
   def self._paste_streams(streams, output, lines = nil, sep = "\t", header = nil, &block)
     output.puts header if header
     streams = streams.collect do |stream|
       if defined? Step and Step === stream
-        stream.get_stream || stream.join.path.open
+        io = stream.get_stream 
+        if io
+          buffer_stream(io)
+        else
+          stream.join.path.open
+        end
       else
         stream
       end
     end
 
     begin
-      done_streams = []
+
       lines ||= streams.collect{|s| s.gets }
       keys = []
       parts = []
@@ -564,6 +582,7 @@ module Misc
       end
       sizes = parts.collect{|p| p.nil? ? 0 : p.length }
       last_min = nil
+
       while lines.compact.any?
         if block_given?
           min = keys.compact.sort(&block).first
@@ -571,14 +590,23 @@ module Misc
           min = keys.compact.sort.first
         end
         str = []
+        threads = []
         keys.each_with_index do |key,i|
           case key
           when min
-            str << [parts[i] * sep]
+            if parts[i] == [SKIP_TAG]
+              str << [sep * (sizes[i]-1)] if sizes[i] > 0
+            else
+              str << [parts[i] * sep]
+            end
+
             line = lines[i] = streams[i].gets
-            if line.nil?
+
+            if line.nil? 
               keys[i] = nil
               parts[i] = nil
+              streams[i].close unless streams[i].closed?
+              streams[i].join if streams[i].respond_to?(:join) 
             else
               k, *p = line.chomp.split(sep, -1)
               keys[i] = k
@@ -589,10 +617,12 @@ module Misc
           end
         end
 
-        output.puts [min, str*sep] * sep
+        output.puts [min, str.flatten*sep] * sep
       end
+
       streams.each do |stream|
-        stream.join if stream.respond_to? :join
+        stream.close unless stream.closed?
+        stream.join if stream.respond_to?(:join) 
       end
     rescue 
       Log.exception $!

data/lib/rbbt/workflow.rb

@@ -607,7 +607,7 @@ module Workflow
   end
 
   def self.load_step(path)
-    Path.setup(path) unless Path === path
+    path = Path.setup(path.dup) unless Path === path
     path = path.find
 
     begin

data/lib/rbbt/workflow/accessor.rb

@@ -35,6 +35,7 @@ module Workflow
                            input_use = rec_input_use(name)
                            input_defaults = rec_input_defaults(name)
                            input_options = rec_input_options(name)
+                           extension = task.extension
                            export = case
                                     when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
                                       :synchronous
@@ -60,7 +61,8 @@ module Workflow
                              :input_use => input_use,
                              :result_type => result_type,
                              :result_description => result_description,
-                             :dependencies => dependencies
+                             :dependencies => dependencies,
+                             :extension => extension
                            }
                          end
   end

data/lib/rbbt/workflow/definition.rb

@@ -18,6 +18,7 @@ module Workflow
     :description        => "",
     :result_type        => nil,
     :result_description => "",
+    :resumable          => false,
     :extension          => nil)
 
 
@@ -33,6 +34,10 @@ module Workflow
     @extension = extension
   end
 
+  def resumable
+    @resumable = true
+  end
+
   def returns(description)
     @result_description = description
   end
@@ -118,6 +123,7 @@ module Workflow
       :input_descriptions => consume_input_descriptions,
       :required_inputs    => consume_required_inputs,
       :extension          => consume_extension,
+      :resumable          => consume_resumable,
       :input_options      => consume_input_options
     }
 

data/lib/rbbt/workflow/remote_workflow/driver/rest.rb

@@ -137,13 +137,15 @@ class RemoteWorkflow
 
         post_thread = Thread.new(Thread.current) do |parent|
           bl = lambda do |rok|
-            if Net::HTTPOK === rok
+            if Net::HTTPOK === rok 
               _url = rok["RBBT-STREAMING-JOB-URL"]
               @url = File.join(task_url, File.basename(_url)) if _url
               rok.read_body do |c,_a, _b|
                 sin.write c
               end
               sin.close
+            elsif Net::HTTPSeeOther === rok
+              raise TryThis.new(rok)
             else
               err = StringIO.new
               rok.read_body do |c,_a, _b|
@@ -156,7 +158,7 @@ class RemoteWorkflow
                        err.rewind
                        err.read
                      end
-              ne = @adaptor.parse_exception text
+              ne = RemoteWorkflow.parse_exception text
               case ne
               when String
                 parent.raise e.class, ne
@@ -173,7 +175,11 @@ class RemoteWorkflow
           end
 
           Log.debug{ "RestClient execute: #{ task_url } - #{Misc.fingerprint task_params}" }
-          RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
+          begin
+            RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
+          rescue TryThis
+            RestClient::Request.execute(:method => :get, :url => $!.payload.header[:location], :block_response => bl)
+          end
         end
 
         # It seems like now response body are now decoded by Net::HTTP after 2.1

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -25,8 +25,13 @@ class RemoteStep < Step
   end
 
   def cache_file
-    digest = Misc.obj2digest([base_url, task, base_name, inputs])
-    Rbbt.var.cache.REST[[task, clean_name, digest] * "."].find
+    begin
+      digest = Misc.obj2digest([base_url, task, base_name, inputs])
+      Rbbt.var.cache.REST[[task, clean_name, digest].compact * "."].find
+    rescue
+      Log.exception $!
+      raise $!
+    end
   end
 
   def cache_files
@@ -62,7 +67,6 @@ class RemoteStep < Step
     no_load ? Misc.add_GET_param(path, "_format", "raw") : @result
   end
 
-
   def self.get_streams(inputs, stream_input = nil)
     new_inputs = {}
     inputs.each do |k,v|
@@ -240,6 +244,7 @@ class RemoteStep < Step
     return true if cache_files.any?
     init_job unless @url
     Log.debug{ "Joining RemoteStep: #{path}" }
+
     if IO === @result
       res = @result
       @result = nil
@@ -253,6 +258,7 @@ class RemoteStep < Step
       sleep 1 unless self.done? || self.aborted? || self.error?
       while not (self.done? || self.aborted? || self.error?)
         sleep 3
+        iif [self.done?, self.status, self.info]
       end
     end
 

data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb

@@ -129,7 +129,7 @@ class RemoteStep
       end
     end
 
-    def _run_job(cache_type = :async)
+    def _run_job(cache_type = :asynchronous)
       get_streams
 
       task_url = URI.encode(File.join(base_url, task.to_s))
@@ -142,7 +142,13 @@ class RemoteStep
       else
         @adaptor.execute_job(base_url, task, task_params, cache_type)
       end
+    end
+
 
+    def produce(*args)
+      @started = true
+      _run_job
     end
+
   end
 end

data/lib/rbbt/workflow/step/accessor.rb

@@ -612,6 +612,10 @@ class Step
     provenance
   end
 
+  def resumable?
+    task && task.resumable
+  end
+
   def config(key, *tokens)
     options = tokens.pop if Hash === tokens.last
     options ||= {}

data/lib/rbbt/workflow/step/dependencies.rb

@@ -92,7 +92,7 @@ class Step
           (job.done? && job.dirty?)  || (job.error? && job.dirty?) ||
           (!(job.noinfo? || job.done? || job.error? || job.aborted? || job.running?))
 
-        job.clean 
+        job.clean unless job.resumable? && (job.updated? && ! job.dirty?)
         job.set_info :status, :cleaned
       end
 
@@ -144,14 +144,18 @@ class Step
 
       dependency.status_lock.synchronize do
         if dependency.aborted? || (dependency.error? && dependency.recoverable_error? && ! canfail_paths.include?(dependency.path) && ! already_failed.include?(dependency.path)) || (!Open.remote?(dependency.path) && dependency.missing?)
-          Log.warn "Cleaning dep. on exec #{Log.color :blue, dependency.path} (missing: #{dependency.missing?}; error #{dependency.error?})"
-          dependency.clean
-          already_failed << dependency.path
-          raise TryAgain
+          if dependency.resumable?
+            dependency.status = :resume
+          else
+            Log.warn "Cleaning dep. on exec #{Log.color :blue, dependency.path} (missing: #{dependency.missing?}; error #{dependency.error?})"
+            dependency.clean
+            already_failed << dependency.path
+            raise TryAgain
+          end
         end
       end
 
-      if ! (dependency.started? || dependency.error?)
+      if dependency.status == :resume || ! (dependency.started? || dependency.error?)
         log_dependency_exec(dependency, :starting)
         dependency.run(true)
         raise TryAgain
@@ -362,7 +366,7 @@ class Step
       (step.dependencies + step.input_dependencies).each do |step_dep|
         next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
-        dep_step[step_dep.path] << step_dep
+        dep_step[step_dep.path] << step
       end
     end
 

data/lib/rbbt/workflow/step/run.rb

@@ -610,7 +610,7 @@ class Step
   end
 
   def _clean_finished
-    if Open.exists? path and not status == :done
+    if Open.exists?(path) && status != :done
       Log.warn "Aborted job had finished. Removing result -- #{ path }"
       begin
         Open.rm path

data/lib/rbbt/workflow/task.rb

@@ -2,7 +2,7 @@ require 'rbbt/util/misc'
 require 'rbbt/persist'
 
 module Task
-  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :required_inputs, :description, :name, :result_description, :extension, :workflow
+  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :required_inputs, :description, :name, :result_description, :extension, :workflow, :resumable
 
   def self.setup(options = {}, &block)
     block.extend Task

data/lib/rbbt/workflow/util/archive.rb

@@ -125,6 +125,7 @@ class Step
   def self.migrate(path, search_path, options = {})
     resource=Rbbt
 
+    orig_path = path
     other_rsync_args = options[:rsync]
 
     recursive = options[:recursive]
@@ -154,6 +155,7 @@ puts files * "\n"
             else
               if File.exists?(path)
                 path = resource.identify(path)
+                raise "Resource #{resource} could not identify #{orig_path}" if path.nil?
               else
                 path = Path.setup(path)
               end
@@ -178,7 +180,7 @@ puts resource[path].find(search_path)
     subpath_files = {}
     paths.sort.each do |path|
       parts = path.split("/")
-      subpath = parts[0..-4] * "/"
+      subpath = parts[0..-4] * "/" + "/"
 
       if subpath_files.keys.any? && subpath.start_with?(subpath_files.keys.last)
         subpath = subpath_files.keys.last
@@ -190,6 +192,7 @@ puts resource[path].find(search_path)
       subpath_files[subpath] << source
     end
 
+    synced_files = []
     subpath_files.each do |subpath, files|
       if options[:target]
         CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
@@ -204,11 +207,15 @@ puts resource[path].find(search_path)
       end
       target = [options[:target], target] * ":" if options[:target]
 
+      next if File.exists?(source) && File.exists?(target) && File.expand_path(source) == File.expand_path(target)
+
       files_and_dirs = Set.new( files )
       files.each do |file|
+        synced_files << File.join(subpath, file)
+
         parts = file.split("/")[0..-2].reject{|p| p.empty?}
         while parts.any?
-          files_and_dirs << parts * "/"
+          files_and_dirs <<  parts * "/"
           parts.pop
         end
       end
@@ -218,8 +225,7 @@ puts resource[path].find(search_path)
 
         cmd = "rsync #{MAIN_RSYNC_ARGS} --progress #{test_str} --files-from='#{tmp_include_file}' #{source}/ #{target}/ #{other_rsync_args}"
 
-        cmd << " && rm -Rf #{source}" if options[:delete]
-
+        #cmd << " && rm -Rf #{source}" if options[:delete]
         if options[:print]
           ppp Open.read(tmp_include_file)
           puts cmd 
@@ -228,6 +234,26 @@ puts resource[path].find(search_path)
         end
       end
     end
+
+    if options[:delete] && synced_files.any?
+      puts Log.color :magenta, "About to erase these files:"
+      synced_files.each do |p|
+        puts Log.color :red, p
+      end 
+
+      if options[:non_interactive]
+        response = 'yes'
+      else
+        puts Log.color :magenta, "Type 'yes' if you are sure:"
+        response = STDIN.gets.chomp
+      end
+
+      if response == 'yes'
+        synced_files.each do |p|
+          Open.rm p
+        end
+      end
+    end
   end
 
   def self.purge(path, recursive = false)

data/share/install/software/lib/install_helpers

@@ -71,7 +71,7 @@ get_pkg(){
     local url="$2"
 
     if [ ! -f "$OPT_SRC_DIR/$name.pkg" ]; then
-        wget "$url" -O "$OPT_SRC_DIR/$name.pkg"  || wget "$url" -O "$OPT_SRC_DIR/$name.pkg" --no-check-certificate
+      wget "$url" -O "$OPT_SRC_DIR/$name.pkg"  || wget "$url" -O "$OPT_SRC_DIR/$name.pkg" --no-check-certificate || (rm -f "$OPT_SRC_DIR/$name.pkg"; echo "Error downloading"; exit -1)
     fi
 }
 
@@ -85,7 +85,7 @@ uncompress_pkg(){
     mkdir -p "$OPT_BUILD_DIR"
     cd "$OPT_BUILD_DIR"
 
-    (tar xvfz $pkg || tar xvfJ $pkg || tar xvfj $pkg || unzip $pkg || echo "Error decompressing") 2> /dev/null
+    (tar xvfz $pkg || tar xvfJ $pkg || tar xvfj $pkg || unzip $pkg || (echo "Error decompressing" & cd & rmdir "$OPT_BUILD_DIR" & exit -1 ) ) 2> /dev/null
 
     cd "$old_pwd"
 }
@@ -207,6 +207,8 @@ prepare_make(){
 
     [ -d src -a ! -e CMakeLists.txt -a ! -e Makefile -a ! -e configure ] && cd src
 
+    [ -f bootstrap ] && (./bootstrap || exit -1)
+
     if [ -f config/m4 ]; then
         libtoolize --force
         aclocal
@@ -275,6 +277,8 @@ build_make(){
         setup "$name"
         clean_build
     fi
+
+    cd "$old_pwd"
 }
 
 buid_cmake(){
@@ -489,7 +493,7 @@ install_jar(){
     local url="$2"
 
     [ -d  "$OPT_DIR/$name/" ] || mkdir -p  "$OPT_DIR/$name/"
-    wget "$url" -O "$OPT_DIR/$name/$name.jar" --no-check-certificate
+    wget "$url" -O "$OPT_DIR/jars/$name.jar" || wget "$url" -O "$OPT_DIR/jars/$name.jar" --no-check-certificate || (rm "$OPT_DIR/jars/$name.jar"; exit -1)
     link "$OPT_DIR/$name/$name.jar" "$OPT_JAR_DIR/$name.jar" 
 }
 

data/share/rbbt_commands/migrate_job

@@ -3,7 +3,7 @@
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
 require 'rbbt/workflow'
-require 'rbbt/workflow/remote/ssh/get'
+require 'rbbt/workflow/remote_workflow'
 
 $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
 
@@ -34,4 +34,6 @@ end
 
 path, search_path, _sep, *other = ARGV
 
+search_path = 'user' if search_path.nil?
+
 Step.migrate(path, search_path, options)

data/share/rbbt_commands/system/status

@@ -187,7 +187,7 @@ workflows.sort.each do |workflow,tasks|
         if info_fields and info_fields.any?
           info = begin
                    Open.open(i[:info_file]) do |f|
-                     Step::INFO_SERIALIAZER.load(f)
+                     Step::INFO_SERIALIZER.load(f)
                    end
                  rescue
                    {:status => :noinfo}
@@ -198,9 +198,10 @@ workflows.sort.each do |workflow,tasks|
       else
         info = begin
                  Open.open(i[:info_file]) do |f|
-                   Step::INFO_SERIALIAZER.load(f)
+                   Step::INFO_SERIALIZER.load(f)
                  end
                rescue
+                 Log.exception $!
                  {:status => :noinfo}
                end
         IndiferentHash.setup(info)

data/share/rbbt_commands/workflow/monitor

@@ -41,7 +41,7 @@ def list_jobs(options)
     clean_file = file.sub('.info','')
     begin
       next if File.exist? clean_file and $quick
-      info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
+      info = Step::INFO_SERIALIZER.load(Open.read(file, :mode => 'rb'))
       next if (File.exist?(clean_file) or info[:status] == :done) and (info[:children_pids].nil? or info[:children_done] or info[:children_pids].select{|pid| Misc.pid_exists? pid}.empty?)
     rescue Exception
       puts "Error parsing info file: #{ file }"
@@ -86,7 +86,7 @@ def clean_jobs(options)
     info = nil
     next if File.exist? clean_file
     begin
-      info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
+      info = Step::INFO_SERIALIZER.load(Open.read(file, :mode => 'rb'))
     rescue Exception
       Log.debug "Error process #{ file }"
       remove_job file if options[:errors]
@@ -117,7 +117,7 @@ begin
     end
   else
     if options[:file]
-      info = Step::INFO_SERIALIAZER.load(Open.read(options[:file], :mode => 'rb'))
+      info = Step::INFO_SERIALIZER.load(Open.read(options[:file], :mode => 'rb'))
       print_job options[:file], info
   else
     list_jobs options

data/test/rbbt/tsv/test_stream.rb

@@ -4,6 +4,7 @@ require 'rbbt/tsv/stream'
 require 'rbbt'
 
 class TestStream < Test::Unit::TestCase
+
   def test_collapse_stream
     text=<<-EOF
 #: :sep=" "
@@ -65,7 +66,7 @@ row3 AAA BBB CCC
 
     text2=<<-EOF
 #: :sep=" "
-#Row Labela Labelb 
+#Row Labela Labelb
 row1 a b
 row3 aaa bbb
 row2 aa bb
@@ -100,7 +101,7 @@ row1 A B C
 
     text2=<<-EOF
 #: :sep=" "
-#Row Labela Labelb 
+#Row Labela Labelb
 row2 aa bb
     EOF
 
@@ -132,7 +133,7 @@ row1 A B C
 
     text2=<<-EOF
 #: :sep=" "
-#Row Labela Labelb 
+#Row Labela Labelb
 row2 aa bb
     EOF
 

data/test/rbbt/util/misc/test_pipes.rb

@@ -285,7 +285,7 @@ line4
       begin
         Misc.consume_stream(sio, false, STDOUT)
       rescue
-        Log.exception $!
+        raise $!
       end
     end
   end

data/test/rbbt/workflow/step/test_dependencies.rb

@@ -4,6 +4,27 @@ require 'rbbt/workflow'
 module DepWorkflow
   extend Workflow
 
+  input :input_file, :file, "Input file", nil, :stream => true
+  task :s1 => :array do |input_file|
+    TSV.traverse input_file, :type => :array, :into => :stream, :bar => "Task1" do |line|
+      line + "\t" << "Task1"
+    end
+  end
+
+  dep :s1
+  task :s2 => :array do |input_file|
+    TSV.traverse step(:s1), :type => :array, :into => :stream, :bar => "Task2" do |line|
+      next [line.split("\t").first, Misc::SKIP_TAG] * "\t" if rand < 0.9
+      line + "\t" << "Task2"
+    end
+  end
+
+  dep :s1
+  dep :s2
+  task :s3 => :array do |input_file|
+    Misc.paste_streams(dependencies.reverse)
+  end
+
   input :input_file, :file, "Input file", nil, :stream => true
   task :task1 => :array do |input_file|
     TSV.traverse input_file, :type => :array, :into => :stream, :bar => "Task1" do |line|
@@ -28,9 +49,7 @@ module DepWorkflow
   dep :task2
   dep :task3
   task :task4 => :array do
-    s1 = TSV.get_stream step(:task2)
-    s2 = TSV.get_stream step(:task3)
-    Misc.paste_streams([s1, s2])
+    Misc.paste_streams(dependencies)
   end
 
   dep :task4
@@ -43,9 +62,7 @@ module DepWorkflow
   dep :task2
   dep :task5
   task :task6 => :array do
-    s1 = TSV.get_stream step(:task2)
-    s2 = TSV.get_stream step(:task5)
-    Misc.paste_streams([s1, s2])
+    Misc.paste_streams(dependencies)
   end
 
   input :stream_file, :file, "Streamed file", nil, :stream => true
@@ -81,6 +98,26 @@ module ComputeWorkflow
 
 end
 
+module ResumeWorkflow
+  extend Workflow
+
+  resumable
+  task :resume => :string do
+    if file('foo').exists?
+      'done'
+    else
+      Open.mkdir files_dir
+      Open.touch(file('foo'))
+      raise 
+    end
+  end
+
+  dep :resume
+  task :reverse => :string do
+    step(:resume).load.reverse
+  end
+end
+
 class TestWorkflowDependency < Test::Unit::TestCase
   def test_task1
     size = 10000
@@ -114,8 +151,25 @@ class TestWorkflowDependency < Test::Unit::TestCase
     end
   end
 
+  def test_task3
+    size = 100000
+    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
+    TmpFile.with_file(content) do |input_file|
+      job = DepWorkflow.job(:task3, "TEST", :input_file => input_file)
+      io = TSV.get_stream job.run(:stream)
+      last_line = nil
+      while line = io.gets
+        last_line = line.strip
+      end
+      io.join
+
+      assert_equal "Line #{size}\tTask1\tTask3", last_line
+    end
+  end
+
   def test_task4
-    size = 1000
+    size = 100000
+    Log.severity = 0
     content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
     TmpFile.with_file(content) do |input_file|
@@ -131,7 +185,7 @@ class TestWorkflowDependency < Test::Unit::TestCase
   end
   
   def test_task5
-    size = 1000
+    size = 10000
     content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
     TmpFile.with_file(content) do |input_file|
@@ -145,20 +199,44 @@ class TestWorkflowDependency < Test::Unit::TestCase
     assert_equal "Line #{size}\tTask1\tTask2\tTask1\tTask3\tTask5", last_line
   end
   
-  def test_task6
+  def test_s3
     size = 100000
-    content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
+    content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
     Log.severity = 0
     TmpFile.with_file(content) do |input_file|
       begin
-      job = DepWorkflow.job(:task6, "TEST", :input_file => input_file)
-      job.run(:stream)
-      io = TSV.get_stream job
-      while line = io.gets
-        last_line = line.strip
+        job = DepWorkflow.job(:s3, "TEST", :input_file => input_file)
+        job.recursive_clean
+        job.run(:stream)
+        io = TSV.get_stream job
+        while line = io.gets
+          last_line = line.strip
+        end
+        io.join if io.respond_to? :join
+      rescue Exception
+        job.abort
+        raise $!
       end
-      io.join
+    end
+    assert last_line.include? "Line #{size}"
+  end
+  
+  def test_task6
+    size = 100000
+    content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
+    last_line = nil
+    Log.severity = 0
+    TmpFile.with_file(content) do |input_file|
+      begin
+        job = DepWorkflow.job(:task6, "TEST", :input_file => input_file)
+        job.recursive_clean
+        job.run(:stream)
+        io = TSV.get_stream job
+        while line = io.gets
+          last_line = line.strip
+        end
+        io.join
       rescue Exception
         job.abort
         raise $!
@@ -168,7 +246,7 @@ class TestWorkflowDependency < Test::Unit::TestCase
   end
   
   def test_task8
-    size = 100000
+    size = 10000
     content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
     last_line = nil
     Log.severity = 0
@@ -204,5 +282,16 @@ class TestWorkflowDependency < Test::Unit::TestCase
       assert_equal "Line #{size}\tTask1\tTask2", last_line
     end
   end
+
+  def test_resume
+    Log.severity = 0
+    job = ResumeWorkflow.job(:reverse)
+    job.recursive_clean
+    assert_raise do
+      job.run
+    end
+    assert job.dependencies.first.file('foo').exists?
+    assert_equal 'done'.reverse, job.run
+  end
 end
 

data/test/rbbt/workflow/test_remote_workflow.rb

@@ -61,6 +61,18 @@ class TestRemoteWorkflow < Test::Unit::TestCase
   def test_rest
     Log.with_severity 0 do
 
+      remote_workflow_server(TestWFRest) do |client|
+        job = client.job(:hi, nil, {})
+        job.clean
+        job = client.job(:hi, nil, {})
+        assert ! job.done?
+        job.run
+        job.produce
+        job = client.job(:hi, nil, {})
+        assert job.done?
+        sleep 1
+      end
+
       remote_workflow_server(TestWFRest) do |client|
         assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
         assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
@@ -78,7 +90,7 @@ class TestRemoteWorkflow < Test::Unit::TestCase
 
   def _test_ssh
     Log.severity = 0
-    client = RemoteWorkflow.new "ssh://turbo:Translation", "Translation"
+    client = RemoteWorkflow.new "ssh://#{ENV["HOSTNAME"]}:Translation", "Translation"
     job = client.job("translate", "SSH-TEST-1", :genes => ["TP53","KRAS"])
     assert_equal 2, job.run.select{|l| l =~ /ENSG/}.length
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.27.2
+  version: 5.27.7
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-29 00:00:00.000000000 Z
+date: 2020-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake