rbbt-util 5.23.21 → 5.23.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '08b49f3913846e398ef9edb5607ffed78089e2ff'
-  data.tar.gz: 775643f2bad7a86d757adbc41c7393a5cbb5f8af
+  metadata.gz: f00bbaf07372fc3c72d60b438ce701b6336e21bb
+  data.tar.gz: f5e9cbffdc19992c1f2d606e7cef8963f84df2b0
 SHA512:
-  metadata.gz: 98ec1677f453ffcaae30565bf6fdc497e1b96bb53dc6972e2ce50a6989aa25b817bc960cb9d4d52e937464c8a11989dc3cd1b53324773d7fa376a0c5f1a7cfb2
-  data.tar.gz: d3b5a410013005ef9ade2b2aa234f895b5c6490090fbbc97b2082e4a8b25e5a4a11031487db3fd6d43d81dde384abb3b5fe565dd55f83dca908b94335067821d
+  metadata.gz: 8d087babf2c286d041e1585db266cd8abd1a39672eab0a965f772b00b9866eafb0ab27bb36242bb4fe4e344733d4f7a5a26033013a20d4f64524daeae0a493db
+  data.tar.gz: 28b6c11840f431d28c6e6bd32b75d72ce6eced2cf12d4f53661e5d316ff893c6c1e9e8d19d73cc78a235798ed9faf64dd389300e9be6e12decf743b335e31266

data/lib/rbbt/persist.rb

@@ -29,8 +29,8 @@ module Persist
     return true if not Open.exists? file
     path = path.find if Path === path
     file = file.find if Path === file
-    patht = File.mtime(path)
-    filet = File.mtime(file)
+    patht = Open.mtime(path)
+    filet = Open.mtime(file)
     diff = patht - filet
     return diff if diff < 0
     return false
@@ -109,7 +109,7 @@ module Persist
       when :string, :text
         Open.read(path)
       when :binary
-        f = File.open(path, 'rb')
+        f = Open.open(path, 'rb')
         res = f.read
         f.close
         res.force_encoding("ASCII-8BIT") if res.respond_to? :force_encoding

data/lib/rbbt/tsv/serializers.rb

@@ -1,3 +1,4 @@
+require 'base64'
 module TSV
 
   class CleanSerializer
@@ -5,6 +6,11 @@ module TSV
     def self.load(o); o end
   end
 
+  class BinarySerializer
+    def self.dump(o); [o].pack('m'); end
+    def self.load(str); str.unpack('m').first; end
+  end
+  
   class IntegerSerializer
     def self.dump(i); [i].pack("l"); end
     def self.load(str); str.unpack("l").first; end
@@ -103,6 +109,7 @@ module TSV
     :flat => StringArraySerializer,
     :double => StringDoubleArraySerializer,
     :clean => CleanSerializer,
+    :binary => BinarySerializer,
     :tsv => TSVSerializer,
     :marshal_tsv => TSVMarshalSerializer
   }

data/lib/rbbt/tsv/util.rb

@@ -106,7 +106,7 @@ module TSV
       end
       file
     when String
-      if Open.remote?(file) or File.exist? file
+      if Open.remote?(file) or Open.exist? file
         Open.open(file, open_options)
       else
         StringIO.new file

data/lib/rbbt/util/log/progress/report.rb

@@ -80,6 +80,7 @@ module Log
         max_history ||= case 
                       when @ticks > 20
                         count = @ticks - @last_count
+                        count = 1 if count == 0
                         if @max
                           times = @max / count
                           num = times / 20

data/lib/rbbt/util/misc/development.rb

@@ -162,6 +162,7 @@ def self.add_libdir(dir=nil)
       end
       raise $!
     rescue Exception
+      Log.exception $! if ENV["RBBT_LOG_INSIST"] == 'true'
       if msg
         Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
       elsif FalseClass === msg

data/lib/rbbt/util/misc/inspect.rb

@@ -45,7 +45,7 @@ module Misc
     when nil
       "nil"
     when (defined? Step and Step)
-      "<Step:"  << (obj.path || Misc.fingerprint([obj.task.name, obj.inputs])) << ">"
+      "<Step:"  << (obj.short_path || Misc.fingerprint([obj.task.name, obj.inputs])) << ">"
     when TrueClass
       "true"
     when FalseClass
@@ -295,7 +295,7 @@ module Misc
               "<IO:" << obj.path << "--" << mtime_str(obj.path) << ">"
             end
           when (defined? Step and Step)
-            "<IO:" << obj.path << ">"
+            "<IO:" << obj.short_path << ">"
           when IO
             if obj.respond_to? :filename and obj.filename
               "<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"

data/lib/rbbt/util/misc/pipes.rb

@@ -304,7 +304,7 @@ module Misc
         into = into.find if Path === into
         if String === into 
           dir = File.dirname(into)
-          FileUtils.mkdir_p dir unless Open.exists?(dir)
+          Open.mkdir dir unless Open.exists?(dir)
           into_path, into = into, Open.open(into, :mode => 'w') 
         end
         into.sync = true if IO === into
@@ -405,10 +405,10 @@ module Misc
               Open.mv tmp_path, path, lock_options
             end
           rescue Exception
-            raise $! unless File.exist? path
+            raise $! unless Open.exists? path
           end
 
-          FileUtils.touch path if File.exist? path
+          Open.touch path if Open.exists? path
           content.join if content.respond_to? :join and not (content.respond_to?(:joined?) and content.joined?)
 
           Open.notify_write(path) 

data/lib/rbbt/util/open.rb

@@ -137,11 +137,14 @@ module Open
     end
   end
 
+  def self.clear_dir_repos
+    @@repos.clear if defined? @@repos and @@repos
+  end
   def self.get_repo_from_dir(dir)
-    @repos ||= {}
-    @repos[dir] ||= begin
+    @@repos ||= {}
+    @@repos[dir] ||= begin
                       repo_path = File.join(dir, '.file_repo')
-                      Persist.open_tokyocabinet(repo_path, false, :clean,TokyoCabinet::BDB )
+                      Persist.open_tokyocabinet(repo_path, false, :clean, TokyoCabinet::BDB )
                     end
   end
 
@@ -149,13 +152,45 @@ module Open
     repo = get_repo_from_dir(dir)
     repo.read_and_close do
       content = repo[sub_path]
-      content.nil? ? nil : StringIO.new(content)
+      content.nil? ? nil : StringIO.new(content).tap{|o| o.binmode }
+    end
+  end
+
+  def self.get_time_from_repo(dir, sub_path)
+    repo = get_repo_from_dir(dir)
+    time = repo.read_and_close do
+      Time.at(repo['.time.' + sub_path].to_i)
+    end
+    time
+  end
+
+  def self.get_atime_from_repo(dir, sub_path)
+    repo = get_repo_from_dir(dir)
+    File.atime(repo.persistance_path)
+  end
+
+  def self.writable_repo?(dir, sub_path)
+    repo = get_repo_from_dir(dir)
+    begin
+      repo.write_and_close do
+      end
+      true
+    rescue
+      false
+    end
+  end
+
+  def self.set_time_from_repo(dir, sub_path)
+    repo = get_repo_from_dir(dir)
+    repo.read_and_close do
+      repo['.time.' + sub_path] = Time.now.to_i.to_s
     end
   end
 
   def self.save_content_in_repo(dir, sub_path, content)
     repo = get_repo_from_dir(dir)
     repo.write_and_close do
+      repo['.time.' + sub_path] = Time.now.to_i.to_s
       repo[sub_path] = content
     end
   end
@@ -174,15 +209,24 @@ module Open
   def self.exists_in_repo(dir, sub_path, content)
     repo = get_repo_from_dir(dir)
     repo.read_and_close do
-      repo.include? sub_path 
+      repo.include?(sub_path) && ! repo[sub_path].nil?
     end
   end
 
   def self.find_repo_dir(file)
     self.repository_dirs.each do |dir|
-      if file.start_with? dir
-        sub_path = file.to_s[dir.length..-1]
-        return [dir, sub_path]
+      dir = dir + '/' unless dir[-1] == "/"
+
+      begin
+        if file.start_with? dir
+          sub_path = file.to_s[dir.length..-1]
+          return [dir, sub_path]
+        else 
+          if Path === file and (ffile = file.find).start_with? dir
+            sub_path = ffile.to_s[dir.length..-1]
+            return [dir, sub_path]
+          end
+        end
       end
     end
     nil
@@ -206,7 +250,21 @@ module Open
 
   def self.file_open(file, grep, mode = 'r', invert_grep = false)
     if (dir_sub_path = find_repo_dir(file))
-      stream = get_stream_from_repo(*dir_sub_path)
+      if mode.include? 'w'
+
+        stream = StringIO.new
+        class << stream
+          attr_accessor :dir_sub_path
+          def close
+            self.rewind
+            Open.save_content_in_repo(*dir_sub_path, self.read)
+          end
+        end
+        stream.dir_sub_path = dir_sub_path
+
+      else
+        stream = get_stream_from_repo(*dir_sub_path)
+      end
     else
       file = file.find if Path === file
       stream =  File.open(file, mode)
@@ -237,9 +295,13 @@ module Open
   end
 
   def self.mkdir(target)
-    target = target.find if Path === target
-    if not File.exists?(target)
-      FileUtils.mkdir_p target
+    if (dir_sub_path = find_repo_dir(target))
+      nil
+    else
+      target = target.find if Path === target
+      if ! File.exists?(target)
+        FileUtils.mkdir_p target
+      end
     end
   end
 
@@ -279,13 +341,47 @@ module Open
     end
   end
 
+  #def self.cp(source, target, options = {})
+  #  source = source.find if Path === source
+  #  target = target.find if Path === target
+
+  #  FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
+  #  FileUtils.rm target if File.exists?(target)
+  #  FileUtils.cp source, target
+  #end
+
   def self.cp(source, target, options = {})
-    source = source.find if Path === source
-    target = target.find if Path === target
+    dir_sub_path_source = find_repo_dir(source)
+    dir_sub_path_target = find_repo_dir(target)
 
-    FileUtils.mkdir_p File.dirname(target) unless File.exists?(File.dirname(target))
-    FileUtils.rm target if File.exists?(target)
-    FileUtils.cp source, target
+    if dir_sub_path_source.nil? and dir_sub_path_target.nil?
+      FileUtils.mkdir_p File.dirname(target) unless File.exist? File.dirname(target)
+      tmp_target = File.join(File.dirname(target), '.tmp_mv.' + File.basename(target))
+      FileUtils.cp source, tmp_target
+      FileUtils.cp tmp_target, target
+      return
+    end
+
+    if dir_sub_path_source.nil?
+      save_content_in_repo(dir_sub_path_target[0], dir_sub_path_target[1], Open.read(source, :mode => 'rb', :nofix => true))
+      return nil
+    end
+
+    if dir_sub_path_target.nil?
+      Open.write(target, get_stream_from_repo(dir_sub_path_source))
+      return nil
+    end
+
+    repo_source = get_repo_from_dir(dir_sub_path_source[0])
+    repo_target = get_repo_from_dir(dir_sub_path_target[0])
+
+    repo_source.read_and_close do
+      repo_target.write_and_close do
+        repo_source[dir_sub_path_source[1]] = repo_target[dir_sub_path_target[1]]
+      end
+    end
+
+    return nil
   end
 
   def self.mv(source, target, options = {})
@@ -301,7 +397,7 @@ module Open
     end
 
     if dir_sub_path_source.nil?
-      save_content_in_repo(dir_sub_path_target[0], dir_sub_path_target[1], Open.read(source))
+      save_content_in_repo(dir_sub_path_target[0], dir_sub_path_target[1], Open.read(source, :mode => 'rb', :nofix => true))
       return nil
     end
 
@@ -317,6 +413,7 @@ module Open
       repo_target.write_and_close do
         repo_source[dir_sub_path_source[1]] = repo_target[dir_sub_path_target[1]]
       end
+      repo_source.delete dir_sub_path_source[1]
     end
 
     return nil
@@ -331,6 +428,9 @@ module Open
       File.exist?(file) || File.symlink?(file)
     end
   end
+  class << self
+    alias exist? exists?
+  end
 
   def self.lock(file, options = {}, &block)
     if file and (dir_sub_path = find_repo_dir(file))
@@ -485,7 +585,7 @@ module Open
   end
 
   def self.can_open?(file)
-    String === file and (File.exist?(file) or remote?(file))
+    String === file and (Open.exist?(file) or remote?(file))
   end
 
   def self.read(file, options = {}, &block)
@@ -511,13 +611,14 @@ module Open
   def self.notify_write(file)
     begin
       notification_file = file + '.notify'
-      if File.exist? notification_file
+      if Open.exists? notification_file
         key = Open.read(notification_file).strip
         key = nil if key.empty?
         Misc.notify("Wrote " << file, nil, key)
-        FileUtils.rm notification_file
+        Open.rm notification_file
       end
     rescue
+      Log.exception $!
       Log.warn "Error notifying write of #{ file }"
     end
   end
@@ -528,48 +629,106 @@ module Open
     file = file.find(options[:where]) if Path === file
     mode = Misc.process_options options, :mode
 
-    FileUtils.mkdir_p File.dirname(file)
-    case
-    when block_given?
-      begin
-        f = File.open(file, mode)
+    if (dir_sub_path = find_repo_dir(file))
+      content = case content
+                when String
+                  content
+                when nil 
+                  if block_given?
+                    yield
+                  else
+                    ""
+                  end
+                else
+                  content.read
+                end
+      dir_sub_path.push content
+      save_content_in_repo(*dir_sub_path)
+    else
+      FileUtils.mkdir_p File.dirname(file)
+      case
+      when block_given?
         begin
-          yield f
-        ensure
-          f.close unless f.closed?
+          f = File.open(file, mode)
+          begin
+            yield f
+          ensure
+            f.close unless f.closed?
+          end
+        rescue Exception
+          FileUtils.rm file if File.exist? file
+          raise $!
         end
-      rescue Exception
-        FileUtils.rm file if File.exist? file
-        raise $!
-      end
-    when content.nil?
-      File.open(file, mode){|f| f.write "" }
-    when String === content
-      file_write(file, content, mode)
-    else
-      begin
-        File.open(file, mode) do |f| 
-          f.flock(File::LOCK_EX)
-          while block = content.read(Misc::BLOCK_SIZE)
-            f.write block
+      when content.nil?
+        File.open(file, mode){|f| f.write "" }
+      when String === content
+        file_write(file, content, mode)
+      else
+        begin
+          File.open(file, mode) do |f| 
+            f.flock(File::LOCK_EX)
+            while block = content.read(Misc::BLOCK_SIZE)
+              f.write block
+            end
+            f.flock(File::LOCK_UN)
           end
-          f.flock(File::LOCK_UN)
+        rescue Exception
+          FileUtils.rm_rf file if File.exist? file
+          raise $!
         end
-      rescue Exception
-        FileUtils.rm_rf file if File.exist? file
-        raise $!
+        content.close
       end
-      content.close
     end
+
     notify_write(file) 
   end
 
   def self.writable?(path)
     path = path.find if Path === path
-    if File.exists?(path)
-      File.writable?(path)
+    if (dir_sub_path = find_repo_dir(path))
+      writable_repo?(*dir_sub_path)
+    else
+      if File.exists?(path)
+        File.writable?(path)
+      else
+        File.writable?(File.dirname(File.expand_path(path)))
+      end
+    end
+  end
+
+  def self.ctime(file)
+    if (dir_sub_path = find_repo_dir(file))
+      get_time_from_repo(*dir_sub_path)
+    else
+      file = file.find if Path === file
+      File.ctime(file)
+    end
+  end
+
+  def self.mtime(file)
+    if (dir_sub_path = find_repo_dir(file))
+      get_time_from_repo(*dir_sub_path)
+    else
+      file = file.find if Path === file
+      File.mtime(file)
+    end
+  end
+
+  def self.atime(file)
+    if (dir_sub_path = find_repo_dir(file))
+      get_atime_from_repo(*dir_sub_path)
     else
-      File.writable?(File.dirname(File.expand_path(path)))
+      file = file.find if Path === file
+      File.atime(file)
+    end
+  end
+
+  def self.touch(file)
+    if (dir_sub_path = find_repo_dir(file))
+      set_time_from_repo(*dir_sub_path)
+    else
+      file = file.find if Path === file
+      FileUtils.touch(file)
     end
   end
 end

data/lib/rbbt/workflow/accessor.rb

@@ -15,7 +15,6 @@ end
 
 class Step
 
-
   INFO_SERIALIAZER = Marshal
 
   def self.wait_for_jobs(jobs)
@@ -64,7 +63,7 @@ class Step
 
   def self.step_info(path)
     begin
-      Open.open(info_file(path)) do |f|
+      Open.open(info_file(path), :mode => 'rb') do |f|
         INFO_SERIALIAZER.load(f)
       end
     rescue Exception
@@ -126,7 +125,7 @@ class Step
     begin
       Misc.insist do
         begin
-          return @info_cache if @info_cache and @info_cache_time and File.ctime(info_file) < @info_cache_time
+          return @info_cache if @info_cache and @info_cache_time and Open.ctime(info_file) < @info_cache_time 
         rescue Exception
           raise $!
         end
@@ -138,7 +137,7 @@ class Step
                 raise TryAgain, "Info locked" if check_lock and info_lock.locked?
                 info_lock.lock if check_lock and false
                 begin
-                  Open.open(info_file) do |file|
+                  Open.open(info_file, :mode => 'rb') do |file|
                     INFO_SERIALIAZER.load(file) #|| {}
                   end
                 ensure
@@ -179,12 +178,8 @@ class Step
       i = info(false).dup
       i[key] = value 
       @info_cache = i
-      Misc.sensiblewrite(info_file, INFO_SERIALIAZER.dump(i), :force => true, :lock => false)
-      Misc.insist do
-        Open.open(info_file) do |file|
-          INFO_SERIALIAZER.load(file) 
-        end
-      end
+      dump = INFO_SERIALIAZER.dump(i)
+      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false)
       @info_cache_time = Time.now
       value
     end
@@ -192,12 +187,14 @@ class Step
 
   def merge_info(hash)
     return nil if @exec or info_file.nil?
+    return nil if ! writable?
     value = Annotated.purge value if defined? Annotated
     Open.lock(info_file, :lock => info_lock) do
       i = info(false)
       i.merge! hash
       @info_cache = i
-      Misc.sensiblewrite(info_file, INFO_SERIALIAZER.dump(i), :force => true, :lock => false)
+      dump = INFO_SERIALIAZER.dump(i)
+      Misc.sensiblewrite(info_file, dump, :force => true, :lock => false)
       @info_cache_time = Time.now
       value
     end
@@ -546,8 +543,8 @@ class Step
     provenance = {}
     dependencies.each do |dep|
       next unless dep.path.exists?
-      if File.exist? dep.info_file
-        provenance[dep.path] = dep.provenance if File.exist? dep.path
+      if Open.exists? dep.info_file
+        provenance[dep.path] = dep.provenance if Open.exists? dep.path
       else
         provenance[dep.path] = nil
       end
@@ -558,7 +555,7 @@ class Step
   def provenance_paths
     provenance = {}
     dependencies.each do |dep|
-      provenance[dep.path] = dep.provenance_paths if File.exist? dep.path
+      provenance[dep.path] = dep.provenance_paths if Open.exists? dep.path
     end
     provenance
   end

data/lib/rbbt/workflow/step/dependencies.rb

@@ -180,7 +180,7 @@ class Step
       raise $!
     rescue Exception
       Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s } -- #{$!.message}"
-      raise $!
+      raise $! unless canfail_paths.include? dependency.path
     end
   end
 
@@ -300,9 +300,9 @@ class Step
   end
 
   def canfail_paths
-    return Set.new if ! File.exists?(info_file)
+    return Set.new if ! Open.exists?(info_file)
 
-    if info[:canfail_paths]
+    if !relocated && info[:canfail_paths] 
       Set.new(info[:canfail_paths])
     else
       canfail_paths = Set.new

data/lib/rbbt/workflow/step/run.rb

@@ -145,7 +145,8 @@ class Step
       next unless dep.updatable?
 
       begin
-        if dep.done? && self.done? && Open.exists?(dep.path) && Open.exists?(self.path) && (File.mtime(dep.path) > File.mtime(self.path))
+
+        if dep.done? && self.done? && Open.exists?(dep.path) && Open.exists?(self.path) && (Open.mtime(dep.path) > Open.mtime(self.path))
           outdated_time << dep
         end
       rescue
@@ -223,7 +224,7 @@ class Step
           begin
             run_dependencies
           rescue Exception
-            FileUtils.rm pid_file if File.exist?(pid_file)
+            Open.rm pid_file if Open.exists?(pid_file)
             stop_dependencies
             raise $!
           end
@@ -346,7 +347,7 @@ class Step
               ensure
                 Step.purge_stream_cache
                 set_info :pid, nil
-                FileUtils.rm pid_file if File.exist?(pid_file)
+                Open.rm pid_file if Open.exist?(pid_file)
               end
             end
 
@@ -361,7 +362,7 @@ class Step
               rescue
                 stop_dependencies
                 set_info :pid, nil
-                FileUtils.rm pid_file if File.exist?(pid_file)
+                Open.rm pid_file if Open.exist?(pid_file)
               end
             end
 
@@ -381,13 +382,13 @@ class Step
             })
             log :ending
             Step.purge_stream_cache
-            FileUtils.rm pid_file if File.exist?(pid_file)
+            Open.rm pid_file if Open.exist?(pid_file)
           end
 
           set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
 
           if result.nil? && File.exists?(self.tmp_path) && ! File.exists?(self.path)
-            FileUtils.mv self.tmp_path, self.path
+            Open.mv self.tmp_path, self.path
           end
           result
         end # END PERSIST
@@ -415,6 +416,8 @@ class Step
       stop_dependencies
       raise $!
     ensure 
+      no_load = false unless IO === result
+      Open.rm pid_file if Open.exist?(pid_file) unless no_load
       set_info :pid, nil unless no_load
     end
   end
@@ -433,6 +436,7 @@ class Step
         else
           e = get_exception
           if e
+            Log.error "Raising exception in produced job #{job.path}: #{e.message}" 
             raise e
           else
             raise "Error in job: #{self.path}"
@@ -464,7 +468,7 @@ class Step
       Misc.pre_fork
       begin
         RbbtSemaphore.wait_semaphore(semaphore) if semaphore
-        FileUtils.mkdir_p File.dirname(path) unless File.exist? File.dirname(path)
+        Open.mkdir File.dirname(path) unless Open.exist?(File.dirname(path))
         begin
           @forked = true
           res = run no_load
@@ -626,7 +630,7 @@ class Step
   end
 
   def soft_grace
-    until done? or File.exist?(info_file)
+    until done? or Open.exist?(info_file)
       sleep 1 
     end
     self
@@ -672,8 +676,8 @@ class Step
         dependencies.each{|dep| dep.join }
       end
 
-      until (path.exists? && status == :done) or error? or aborted? or waiting?
-        sleep 1 
+      until (Open.exists?(path) && status == :done) or error? or aborted? or waiting?
+        sleep 1
         join_stream if streaming?
       end
 
@@ -682,7 +686,7 @@ class Step
       begin
         set_info :joined, true 
       rescue
-      end if File.exists?(info_file) && writable?
+      end if Open.exists?(info_file) && writable?
       @result = nil
     end
   end

data/lib/rbbt/workflow/step.rb

@@ -9,6 +9,7 @@ class Step
   attr_accessor :task_name, :overriden
   attr_accessor :pid
   attr_accessor :exec
+  attr_accessor :relocated
   attr_accessor :result, :mutex, :seen
 
   class << self
@@ -252,19 +253,29 @@ class Step
 
 
   def load
-    res = if @result and not @path == @result
-            res = @result
-          else
-            join if not done?
-            @path.exists? ? Persist.load_file(@path, result_type) : exec
+    res = begin
+            if @result and not @path == @result
+              res = @result
+            else
+              join if not done?
+              res = @path.exists? ? Persist.load_file(@path, result_type) : exec
+            end
+
+            if result_description
+              entity_info = info.dup
+              entity_info.merge! info[:inputs] if info[:inputs]
+              res = prepare_result res, result_description, entity_info 
+            end
+
+            res
+          rescue IOError
+            if @result
+              @result = nil
+              retry
+            end
+            raise $!
           end
 
-    if result_description
-      entity_info = info.dup
-      entity_info.merge! info[:inputs] if info[:inputs]
-      res = prepare_result res, result_description, entity_info 
-    end
-
     res
   end
 

data/lib/rbbt/workflow.rb

@@ -471,21 +471,24 @@ module Workflow
     preal = real.split(/\/+/)
     pother = other.split(/\/+/)
     end_part = pother[-3..-1] * "/"
-    new_path = preal[0..-4] * "/" << end_part
+    new_path = preal[0..-4] * "/" << "/" << end_part
     return new_path if File.exists? new_path
     Rbbt.var.jobs[end_part].find
   end
 
   def self.load_step(path)
     step = Step.new path
+    relocated = false
     step.dependencies = (step.info[:dependencies] || []).collect do |task,name,dep_path|
-      if File.exists?(dep_path)
+      if Open.exists?(dep_path)
         Workflow.load_step dep_path
       else
         new_path = relocate(path, dep_path)
+        relocated = true if Open.exists?(new_path)
         Workflow.load_step new_path
       end
     end
+    step.relocated = relocated
 
     step
   end

data/share/rbbt_commands/system/clean

@@ -103,6 +103,14 @@ TSV.traverse jobs, :_bar => "Checking job status" do |file,i|
     status = :sync if status != "done" and File.exist? file
 
     status = :dirty if info[:status].to_s == "done" and dirty and Workflow.load_step(file).dirty?
+    if info[:status] == :error
+      begin
+        exception = info[:exception][:class]
+        Kernel.const_get exception
+        status = :non_recoverable if exception.superclass === RbbtException
+      rescue
+      end
+    end
 
     status = status.to_s
   end

data/share/rbbt_commands/workflow/example

@@ -88,6 +88,8 @@ orig_name = name
 
 $saved_args = ARGV.dup
 
+$saved_args.shift if $saved_args.include? '--'
+
 workflow = Workflow.require_workflow workflow
 
 tasks = task ? [task] : workflow.libdir.examples.glob('*').collect{|file| File.basename file }

data/share/rbbt_commands/workflow/prov

@@ -70,7 +70,7 @@ def report_msg(status, name, path, info = nil)
     task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:blue, "file"), Log.color(:magenta, "-")
   end
 
-  str = if not Open.remote?(path) and (File.exists?(path) and $main_mtime and ($main_mtime - File.mtime(path)) < 0)
+  str = if not Open.remote?(path) and (Open.exists?(path) and $main_mtime and ($main_mtime - Open.mtime(path)) < 0)
     status_msg(status.to_s) << " " << [workflow, task, path].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
   else
     status_msg(status.to_s) << " " << [workflow, task, path].compact * " " 
@@ -108,7 +108,7 @@ def report(step, offset = 0, task = nil)
   status = info[:status] || :missing
   status = "remote" if Open.remote?(path)
   name = info[:name] || File.basename(path)
-  status = :unsync if status == :done and not File.exist? path
+  status = :unsync if status == :done and not Open.exist? path
   str = " " * offset
   str << report_msg(status, name, path, info)
   step.dependencies.each do |dep|
@@ -124,7 +124,7 @@ def report(step, offset = 0, task = nil)
 end
 
 step = get_step file
-$main_mtime = File.exist?(step.path) ? File.mtime(step.path) : nil
+$main_mtime = Open.exist?(step.path) ? Open.mtime(step.path) : nil
 
 def adjacency(step)
 
@@ -138,7 +138,7 @@ def adjacency(step)
     workflow, task, name = path.split("/")[-3..-1]
   end
   name = name.split(":").last
-  status = :unsync if status == :done and not File.exist? path
+  status = :unsync if status == :done and not Open.exist? path
   shapes = Hash.new "circle"
   edge_info = {:status =>  status, :workflow => workflow, :task => task, :name => name, :label => task, :shape => shapes[workflow], :color => status == 'remote' ? 'blue' : 'green'}
   id = Misc.digest(path)

data/test/rbbt/test_workflow.rb

@@ -129,12 +129,64 @@ for this dependency
     "Hi #{name}"
   end
 
+  input :num, :integer
+  task :odd => :integer do |num|
+    raise ParameterException, "Not odd" if num % 2 == 0
+    num
+  end
+
+  dep :odd, :num => 10, :compute => :canfail 
+  dep :odd, :num => 11, :compute => :canfail 
+  dep :odd, :num => 12, :compute => :canfail 
+  dep :odd, :num => 13, :compute => :canfail 
+  task :sum_odds => :integer do 
+    dependencies.inject(0) do |acc, dep|
+      acc += dep.load unless dep.error?
+      acc
+    end
+  end
+
+  dep :sum_odds
+  task :sum_odds_str => :string do
+    "Sum odds: " << step(:sum_odds).load.to_s
+  end
+
+
 end
 
 TestWF.workdir = Rbbt.tmp.test.workflow
 
 class TestWorkflow < Test::Unit::TestCase
 
+  
+  def test_repo_marshal
+    TmpFile.with_file do |tmpdir|
+      tmpdir = Rbbt.tmp.repo_dir.find
+      repo = File.join(tmpdir, 'repo')
+
+      filename = 'file'
+      Open.repository_dirs.push(repo)
+
+      job = TestWF.job(:call_name, "Miguel")
+      job.run
+
+      obj = job.info
+      Open.write(File.join(repo, filename), Marshal.dump(obj))
+      new =Open.open(File.join(repo, filename)) do |f|
+        Marshal.load(f)
+      end
+
+      assert_equal new, obj
+    end
+
+  end
+  
+  def test_in_repo
+    job = TestWF.job(:call_name, "Miguel")
+    assert_equal "Hi Miguel", job.run
+    assert_equal "Miguel", job.clean_name
+  end
+
   def test_as_jobname
     job = TestWF.job(:call_name, "Miguel")
     assert_equal "Hi Miguel", job.run
@@ -146,21 +198,22 @@ class TestWorkflow < Test::Unit::TestCase
   end
 
   def test_update_on_input_dependency_update
+    Open.repository_dirs << File.join(ENV["HOME"],".rbbt/tmp/test/workflow")
     Log.severity = 0
     send_input_dep_to_reverse_job = TestWF.job(:send_input_dep_to_reverse, nil, :name => "Miguel")
     send_input_dep_to_reverse_job.clean
     send_input_dep_to_reverse_job.run
 
     input_dep_job = send_input_dep_to_reverse_job.step(:input_dep)
-    mtime_orig = File.mtime send_input_dep_to_reverse_job.step(:reverse_input_text).path
+    mtime_orig = Open.mtime send_input_dep_to_reverse_job.step(:reverse_input_text).path
 
-    sleep 1
+    sleep 2
     input_dep_job.clean
     input_dep_job.run
     send_input_dep_to_reverse_job = TestWF.job(:send_input_dep_to_reverse, nil, :name => "Miguel")
 
     send_input_dep_to_reverse_job.run
-    mtime_new = File.mtime send_input_dep_to_reverse_job.step(:reverse_input_text).path
+    mtime_new = Open.mtime send_input_dep_to_reverse_job.step(:reverse_input_text).path
     assert mtime_orig < mtime_new
   end
 
@@ -170,7 +223,7 @@ class TestWorkflow < Test::Unit::TestCase
 
   def test_job
     str = "TEST"
-    job = TestWF.job(:repeat2, "Default", :number => 3).fork
+    job = TestWF.job(:repeat2, "Default", :number => 3).clean.fork
     while not job.done?
       sleep 1
     end
@@ -275,16 +328,6 @@ class TestWorkflow < Test::Unit::TestCase
     assert_equal "CB", TestWF.job(:t3).run
   end
 
-  def test_relocate
-    listed = '/home/user/.rbbt/var/jobs/TestWF/task1/Default'
-    real = '/usr/local/var/rbbt/jobs/TestWF/task1/Default'
-    other = '/home/user/.rbbt/var/jobs/TestWF/task2/Default'
-    real_other = '/usr/local/var/rbbt/jobs/TestWF/task2/Default'
-
-    assert_equal real_other, Workflow.relocate(real, other)
-    assert_equal real_other, Workflow.relocate(real, other)
-  end
-
   def test_transplant
     listed = '/home/user/.rbbt/var/jobs/TestWF/task1/Default'
     real = '/usr/local/var/rbbt/jobs/TestWF/task1/Default'
@@ -295,13 +338,29 @@ class TestWorkflow < Test::Unit::TestCase
     assert_equal real_other, Workflow.transplant(nil, real, other)
   end
 
-  def test_relocate_alt
-    listed = '/scratch/tmp/rbbt/.rbbt/var/jobs/Study/sample_gene_cnvs_focal/Bladder-TCC'
-    real = '/home/bsc26/bsc26892/.rbbt/var/jobs/Study/sample_gene_cnvs_focal/Bladder-TCC'
-    other = '/scratch/tmp/rbbt/scratch/bsc26892/rbbt/var/jobs/Sample/gene_cnv_status_focal/PCAWG'
-    real_other = '/home/bsc26/bsc26892/.rbbt/var/jobs/Sample/gene_cnv_status_focal/PCAWG'
+  def test_relocate
+    TmpFile.with_file do |tmpdir|
+      listed = File.join(tmpdir, '/home/user/.rbbt/var/jobs/TestWF/task1/Default')
+      real = File.join(tmpdir, '/usr/local/var/rbbt/jobs/TestWF/task1/Default')
+      other = File.join(tmpdir, '/home/user/.rbbt/var/jobs/TestWF/task2/Default')
+      real_other = File.join(tmpdir, '/usr/local/var/rbbt/jobs/TestWF/task2/Default')
+
+      Open.write(real_other,'')
+      assert_equal real_other, Workflow.relocate(real, other)
+      assert_equal real_other, Workflow.relocate(real, other)
+    end
+  end
 
-    assert_equal real_other, Workflow.relocate(real, other)
+  def test_relocate_alt
+    TmpFile.with_file do |tmpdir|
+      listed = File.join(tmpdir, '/scratch/tmp/rbbt/.rbbt/var/jobs/Study/sample_gene_cnvs_focal/Bladder-TCC')
+      real = File.join(tmpdir, '/home/bsc26/bsc26892/.rbbt/var/jobs/Study/sample_gene_cnvs_focal/Bladder-TCC')
+      other = File.join(tmpdir, '/scratch/tmp/rbbt/scratch/bsc26892/rbbt/var/jobs/Sample/gene_cnv_status_focal/PCAWG')
+      real_other = File.join(tmpdir, '/home/bsc26/bsc26892/.rbbt/var/jobs/Sample/gene_cnv_status_focal/PCAWG')
+      Open.write(real_other,'')
+
+      assert_equal real_other, Workflow.relocate(real, other)
+    end
   end
 
   def test_delete_dep
@@ -317,8 +376,17 @@ class TestWorkflow < Test::Unit::TestCase
     job = TestWF.job(:t3)
     job.step(:t1).clean
     Misc.with_env "RBBT_UPDATE_ALL_JOBS", "true" do
-      assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
+     assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
     end
   end
 
+  def test_canfail
+    job = TestWF.job(:sum_odds)
+    assert_equal 24, job.run
+
+    job = TestWF.job(:sum_odds_str)
+    job.recursive_clean
+    assert_equal "Sum odds: 24", job.run
+  end
+
 end

data/test/rbbt/util/test_open.rb

@@ -109,7 +109,7 @@ class TestOpen < Test::Unit::TestCase
     file1 = "TEST"
     file2 = "TEST" * 1000
     TmpFile.with_file do |tmpdir|
-      tmpdir = "/home/mvazquezg/tmp/repo_dir"
+      tmpdir = Rbbt.tmp.repo_dir.find
       normal = File.join(tmpdir, 'normal')
       repo = File.join(tmpdir, 'repo')
 
@@ -135,7 +135,7 @@ class TestOpen < Test::Unit::TestCase
   
   def test_repo_dir2
     TmpFile.with_file do |tmpdir|
-      tmpdir = "/home/mvazquezg/tmp/repo_dir"
+      tmpdir = Rbbt.tmp.repo_dir.find
       repo = File.join(tmpdir, 'repo')
 
       Open.repository_dirs.push(repo)
@@ -149,5 +149,44 @@ class TestOpen < Test::Unit::TestCase
     end
   end
 
+  def test_repo_marshal
+    TmpFile.with_file do |tmpdir|
+      tmpdir = Rbbt.tmp.repo_dir.find
+      repo = File.join(tmpdir, 'repo')
+
+      filename = 'file'
+      Open.repository_dirs.push(repo)
+
+      obj = {:a => "string", :pid => nil, :num => 1000, :p  => Rbbt.tmp.foo}
+      Open.write(File.join(repo, filename), Marshal.dump(obj))
+      new =Open.open(File.join(repo, filename)) do |f|
+        Marshal.load(f)
+      end
+
+      assert_equal new, obj
+    end
+
+  end
+
+  def test_write_stream_repo
+    Log.severity = 0
+    TmpFile.with_file do |tmpdir|
+      tmpdir = Rbbt.tmp.repo_dir.find
+      repo = File.join(tmpdir, 'repo')
+
+      file = File.join(repo, 'file')
+      Open.repository_dirs.push(repo)
+
+      text = (["text"] * 5) * "\n"
+
+      Misc.consume_stream(StringIO.new(text), false, file)
+
+      assert_equal text, Open.read(file)
+      assert !File.exists?(file)
+      assert Open.exists? file
+    end
+
+  end
+  
 end
 

data/test/test_helper.rb

@@ -25,6 +25,7 @@ class Test::Unit::TestCase
 
   def teardown
     FileUtils.rm_rf Rbbt.tmp.test.workflow.find
+    #Open.clear_dir_repos
     #if defined? Persist
     #  FileUtils.rm_rf Path.setup("", 'rbbt').tmp.test.find :user
     #  Persist::CONNECTIONS.values.each do |c| c.close end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.23.21
+  version: 5.23.22
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-08-27 00:00:00.000000000 Z
+date: 2018-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake