rbbt-util 5.27.5 → 5.27.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8792f1d19116c0384fd0063a5e81c8e9d9e30026e5dbaf41d5b26cf56210aa8
-  data.tar.gz: d3c1cf57d737a5255c6f31eb3ebe907c21d8449a0fae89d0208bb2935b513903
+  metadata.gz: fe7c3dec4d2333e77ea396a2d6e572512bf7033999ff4715f1a1c6c9d3d1b9d5
+  data.tar.gz: 32a3d905c1a95f9c7e5cb359855f8781509cfeecdd8db7b77ee29eaea528d030
 SHA512:
-  metadata.gz: d5f84745610ef5f21c23f193b17b2b16f7b4a6a17218e4e5a0d3e23fb68896d9039a99e6d72bf13952b2e39fb5b96b9809ca8683efdb788084c7ef688bac1c13
-  data.tar.gz: 3f17849b28cfe2fd8618b72a74a9887172f90d93876ec59bbaae5d6fec4591537410958fcd5910805a6c995ebdef81e566d6056803090d8625efaccd0a3e2588
+  metadata.gz: d009024642898f775d423e1ddede6aea538e8ed5ff4675eedadd64768e67b0a46f6519038378a221db76b0e196cfc713ca8dc7553788cdb9a51014c14a97de4c
+  data.tar.gz: 5c0612feca84b00e6d402f7a68ca1cebd6f4b7a785b40899bdaed3385a3f68853824dcc98d9cfd2e641768ef5ca3d52e38e41bdbc1d7a3751ae288e8f8d7ed2d

data/lib/rbbt/fix_width_table.rb

@@ -4,20 +4,21 @@ class FixWidthTable
   def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
     @filename = filename
 
-    if update or %w(memory stringio).include?(filename.to_s.downcase) or not File.exist?(filename)
+    if update || %w(memory stringio).include?(filename.to_s.downcase) || ! File.exist?(filename)
       Log.debug "FixWidthTable create: #{ filename }"
       @value_size  = value_size
       @range       = range
       @record_size = @value_size + (@range ? 16 : 8)
       @write = true
 
-      if %w(memory stringio).include? filename.to_s.downcase
+      if %w(memory stringio).include?(filename.to_s.downcase)
         @filename = :memory
         @file     = StringIO.new
       else
         FileUtils.rm @filename if File.exist? @filename
         FileUtils.mkdir_p File.dirname(@filename) unless File.exist? @filename
-        @file = File.open(@filename, 'wb')
+        #@file = File.open(@filename, 'wb')
+        @file = File.open(@filename, 'w:ASCII-8BIT')
       end
 
       @file.write [value_size].pack("L")
@@ -25,9 +26,9 @@ class FixWidthTable
 
       @size = 0
     else
-      Log.debug "FixWidthTable up-to-date: #{ filename }"
+      Log.debug "FixWidthTable up-to-date: #{ filename } - (in_memory:#{in_memory})"
       if in_memory
-        @file        = StringIO.new(Open.read(@filename, :mode => 'r:ASCII-8BIT'), 'r')
+        @file        = Open.open(@filename, :mode => 'r:ASCII-ASCII'){|f| StringIO.new f.read}
       else
         @file        = File.open(@filename, 'r:ASCII-8BIT')
       end

data/lib/rbbt/persist/tsv/tokyocabinet.rb

@@ -16,7 +16,7 @@ module Persist
       end
 
       dir = File.dirname(File.expand_path(path))
-      Open.mkdir(dir) unless Open.exists?(dir)
+      File.mkdir(dir) unless File.exists?(dir)
 
       tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
       tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
@@ -87,7 +87,7 @@ module Persist
   def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
     write = true unless File.exist? path
 
-    FileUtils.mkdir_p File.dirname(path) unless Open.exist?(File.dirname(path))
+    FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
 
     database = Persist::TCAdapter.open(path, write, tokyocabinet_class)
 

data/lib/rbbt/resource/path.rb

@@ -199,7 +199,7 @@ module Path
                          next if res
                          next unless paths.include? w
                          path = find(w, caller_lib, paths)
-                         res = path if File.exist? path
+                         res = path if File.exist?(path)
                        end if res.nil?
 
                        (paths.keys - STANDARD_SEARCH - search_order).each do |w|
@@ -241,8 +241,8 @@ module Path
                          sub('{REMOVE}/', '').
                          sub('{REMOVE}', '')
 
-                       path = path + '.gz' if File.exist? path + '.gz'
-                       path = path + '.bgz' if File.exist? path + '.bgz'
+                       path = path + '.gz' if File.exist?(path + '.gz')
+                       path = path + '.bgz' if File.exist?(path + '.bgz')
 
                        self.annotate path
 

data/lib/rbbt/util/misc/exceptions.rb

@@ -15,6 +15,14 @@ end
 class Aborted < StandardError; end
 
 class TryAgain < StandardError; end
+
+class TryThis < StandardError
+  attr_accessor :payload
+  def initialize(payload = nil)
+    @payload = payload
+  end
+end
+
 class SemaphoreInterrupted < TryAgain; end
 class LockInterrupted < TryAgain; end
 

data/lib/rbbt/workflow/definition.rb

@@ -18,6 +18,7 @@ module Workflow
     :description        => "",
     :result_type        => nil,
     :result_description => "",
+    :resumable          => false,
     :extension          => nil)
 
 
@@ -33,6 +34,10 @@ module Workflow
     @extension = extension
   end
 
+  def resumable
+    @resumable = true
+  end
+
   def returns(description)
     @result_description = description
   end
@@ -118,6 +123,7 @@ module Workflow
       :input_descriptions => consume_input_descriptions,
       :required_inputs    => consume_required_inputs,
       :extension          => consume_extension,
+      :resumable          => consume_resumable,
       :input_options      => consume_input_options
     }
 

data/lib/rbbt/workflow/remote_workflow/driver/rest.rb

@@ -137,13 +137,15 @@ class RemoteWorkflow
 
         post_thread = Thread.new(Thread.current) do |parent|
           bl = lambda do |rok|
-            if Net::HTTPOK === rok
+            if Net::HTTPOK === rok 
               _url = rok["RBBT-STREAMING-JOB-URL"]
               @url = File.join(task_url, File.basename(_url)) if _url
               rok.read_body do |c,_a, _b|
                 sin.write c
               end
               sin.close
+            elsif Net::HTTPSeeOther === rok
+              raise TryThis.new(rok)
             else
               err = StringIO.new
               rok.read_body do |c,_a, _b|
@@ -156,7 +158,7 @@ class RemoteWorkflow
                        err.rewind
                        err.read
                      end
-              ne = @adaptor.parse_exception text
+              ne = RemoteWorkflow.parse_exception text
               case ne
               when String
                 parent.raise e.class, ne
@@ -173,7 +175,11 @@ class RemoteWorkflow
           end
 
           Log.debug{ "RestClient execute: #{ task_url } - #{Misc.fingerprint task_params}" }
-          RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
+          begin
+            RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
+          rescue TryThis
+            RestClient::Request.execute(:method => :get, :url => $!.payload.header[:location], :block_response => bl)
+          end
         end
 
         # It seems like now response body are now decoded by Net::HTTP after 2.1

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -25,8 +25,13 @@ class RemoteStep < Step
   end
 
   def cache_file
-    digest = Misc.obj2digest([base_url, task, base_name, inputs])
-    Rbbt.var.cache.REST[[task, clean_name, digest] * "."].find
+    begin
+      digest = Misc.obj2digest([base_url, task, base_name, inputs])
+      Rbbt.var.cache.REST[[task, clean_name, digest].compact * "."].find
+    rescue
+      Log.exception $!
+      raise $!
+    end
   end
 
   def cache_files
@@ -62,7 +67,6 @@ class RemoteStep < Step
     no_load ? Misc.add_GET_param(path, "_format", "raw") : @result
   end
 
-
   def self.get_streams(inputs, stream_input = nil)
     new_inputs = {}
     inputs.each do |k,v|
@@ -240,6 +244,7 @@ class RemoteStep < Step
     return true if cache_files.any?
     init_job unless @url
     Log.debug{ "Joining RemoteStep: #{path}" }
+
     if IO === @result
       res = @result
       @result = nil
@@ -253,6 +258,7 @@ class RemoteStep < Step
       sleep 1 unless self.done? || self.aborted? || self.error?
       while not (self.done? || self.aborted? || self.error?)
         sleep 3
+        iif [self.done?, self.status, self.info]
       end
     end
 

data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb

@@ -129,7 +129,7 @@ class RemoteStep
       end
     end
 
-    def _run_job(cache_type = :async)
+    def _run_job(cache_type = :asynchronous)
       get_streams
 
       task_url = URI.encode(File.join(base_url, task.to_s))
@@ -142,7 +142,13 @@ class RemoteStep
       else
         @adaptor.execute_job(base_url, task, task_params, cache_type)
       end
+    end
+
 
+    def produce(*args)
+      @started = true
+      _run_job
     end
+
   end
 end

data/lib/rbbt/workflow/step/accessor.rb

@@ -612,6 +612,10 @@ class Step
     provenance
   end
 
+  def resumable?
+    task && task.resumable
+  end
+
   def config(key, *tokens)
     options = tokens.pop if Hash === tokens.last
     options ||= {}

data/lib/rbbt/workflow/step/dependencies.rb

@@ -92,7 +92,7 @@ class Step
           (job.done? && job.dirty?)  || (job.error? && job.dirty?) ||
           (!(job.noinfo? || job.done? || job.error? || job.aborted? || job.running?))
 
-        job.clean 
+        job.clean unless job.resumable? && (job.updated? && ! job.dirty?)
         job.set_info :status, :cleaned
       end
 
@@ -144,14 +144,18 @@ class Step
 
       dependency.status_lock.synchronize do
         if dependency.aborted? || (dependency.error? && dependency.recoverable_error? && ! canfail_paths.include?(dependency.path) && ! already_failed.include?(dependency.path)) || (!Open.remote?(dependency.path) && dependency.missing?)
-          Log.warn "Cleaning dep. on exec #{Log.color :blue, dependency.path} (missing: #{dependency.missing?}; error #{dependency.error?})"
-          dependency.clean
-          already_failed << dependency.path
-          raise TryAgain
+          if dependency.resumable?
+            dependency.status = :resume
+          else
+            Log.warn "Cleaning dep. on exec #{Log.color :blue, dependency.path} (missing: #{dependency.missing?}; error #{dependency.error?})"
+            dependency.clean
+            already_failed << dependency.path
+            raise TryAgain
+          end
         end
       end
 
-      if ! (dependency.started? || dependency.error?)
+      if dependency.status == :resume || ! (dependency.started? || dependency.error?)
         log_dependency_exec(dependency, :starting)
         dependency.run(true)
         raise TryAgain

data/lib/rbbt/workflow/step/run.rb

@@ -610,7 +610,7 @@ class Step
   end
 
   def _clean_finished
-    if Open.exists? path and not status == :done
+    if Open.exists?(path) && status != :done
       Log.warn "Aborted job had finished. Removing result -- #{ path }"
       begin
         Open.rm path

data/lib/rbbt/workflow/task.rb

@@ -2,7 +2,7 @@ require 'rbbt/util/misc'
 require 'rbbt/persist'
 
 module Task
-  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :required_inputs, :description, :name, :result_description, :extension, :workflow
+  attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :required_inputs, :description, :name, :result_description, :extension, :workflow, :resumable
 
   def self.setup(options = {}, &block)
     block.extend Task

data/share/rbbt_commands/system/status

@@ -187,7 +187,7 @@ workflows.sort.each do |workflow,tasks|
         if info_fields and info_fields.any?
           info = begin
                    Open.open(i[:info_file]) do |f|
-                     Step::INFO_SERIALIAZER.load(f)
+                     Step::INFO_SERIALIZER.load(f)
                    end
                  rescue
                    {:status => :noinfo}
@@ -198,9 +198,10 @@ workflows.sort.each do |workflow,tasks|
       else
         info = begin
                  Open.open(i[:info_file]) do |f|
-                   Step::INFO_SERIALIAZER.load(f)
+                   Step::INFO_SERIALIZER.load(f)
                  end
                rescue
+                 Log.exception $!
                  {:status => :noinfo}
                end
         IndiferentHash.setup(info)

data/share/rbbt_commands/workflow/monitor

@@ -41,7 +41,7 @@ def list_jobs(options)
     clean_file = file.sub('.info','')
     begin
       next if File.exist? clean_file and $quick
-      info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
+      info = Step::INFO_SERIALIZER.load(Open.read(file, :mode => 'rb'))
       next if (File.exist?(clean_file) or info[:status] == :done) and (info[:children_pids].nil? or info[:children_done] or info[:children_pids].select{|pid| Misc.pid_exists? pid}.empty?)
     rescue Exception
       puts "Error parsing info file: #{ file }"
@@ -86,7 +86,7 @@ def clean_jobs(options)
     info = nil
     next if File.exist? clean_file
     begin
-      info = Step::INFO_SERIALIAZER.load(Open.read(file, :mode => 'rb'))
+      info = Step::INFO_SERIALIZER.load(Open.read(file, :mode => 'rb'))
     rescue Exception
       Log.debug "Error process #{ file }"
       remove_job file if options[:errors]
@@ -117,7 +117,7 @@ begin
     end
   else
     if options[:file]
-      info = Step::INFO_SERIALIAZER.load(Open.read(options[:file], :mode => 'rb'))
+      info = Step::INFO_SERIALIZER.load(Open.read(options[:file], :mode => 'rb'))
       print_job options[:file], info
   else
     list_jobs options

data/test/rbbt/workflow/step/test_dependencies.rb

@@ -40,8 +40,8 @@ module DepWorkflow
     end
   end
 
-  dep :task2
   dep :task5
+  dep :task2
   task :task6 => :array do
     s1 = TSV.get_stream step(:task2)
     s2 = TSV.get_stream step(:task5)
@@ -81,6 +81,26 @@ module ComputeWorkflow
 
 end
 
+module ResumeWorkflow
+  extend Workflow
+
+  resumable
+  task :resume => :string do
+    if file('foo').exists?
+      'done'
+    else
+      Open.mkdir files_dir
+      Open.touch(file('foo'))
+      raise 
+    end
+  end
+
+  dep :resume
+  task :reverse => :string do
+    step(:resume).load.reverse
+  end
+end
+
 class TestWorkflowDependency < Test::Unit::TestCase
   def test_task1
     size = 10000
@@ -152,13 +172,14 @@ class TestWorkflowDependency < Test::Unit::TestCase
     Log.severity = 0
     TmpFile.with_file(content) do |input_file|
       begin
-      job = DepWorkflow.job(:task6, "TEST", :input_file => input_file)
-      job.run(:stream)
-      io = TSV.get_stream job
-      while line = io.gets
-        last_line = line.strip
-      end
-      io.join
+        job = DepWorkflow.job(:task6, "TEST", :input_file => input_file)
+        job.recursive_clean
+        job.run(:stream)
+        io = TSV.get_stream job
+        while line = io.gets
+          last_line = line.strip
+        end
+        io.join
       rescue Exception
         job.abort
         raise $!
@@ -204,5 +225,16 @@ class TestWorkflowDependency < Test::Unit::TestCase
       assert_equal "Line #{size}\tTask1\tTask2", last_line
     end
   end
+
+  def test_resume
+    Log.severity = 0
+    job = ResumeWorkflow.job(:reverse)
+    job.recursive_clean
+    assert_raise do
+      job.run
+    end
+    assert job.dependencies.first.file('foo').exists?
+    assert_equal 'done'.reverse, job.run
+  end
 end
 

data/test/rbbt/workflow/test_remote_workflow.rb

@@ -61,6 +61,18 @@ class TestRemoteWorkflow < Test::Unit::TestCase
   def test_rest
     Log.with_severity 0 do
 
+      remote_workflow_server(TestWFRest) do |client|
+        job = client.job(:hi, nil, {})
+        job.clean
+        job = client.job(:hi, nil, {})
+        assert ! job.done?
+        job.run
+        job.produce
+        job = client.job(:hi, nil, {})
+        assert job.done?
+        sleep 1
+      end
+
       remote_workflow_server(TestWFRest) do |client|
         assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
         assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
@@ -78,7 +90,7 @@ class TestRemoteWorkflow < Test::Unit::TestCase
 
   def _test_ssh
     Log.severity = 0
-    client = RemoteWorkflow.new "ssh://turbo:Translation", "Translation"
+    client = RemoteWorkflow.new "ssh://#{ENV["HOSTNAME"]}:Translation", "Translation"
     job = client.job("translate", "SSH-TEST-1", :genes => ["TP53","KRAS"])
     assert_equal 2, job.run.select{|l| l =~ /ENSG/}.length
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.27.5
+  version: 5.27.6
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-12 00:00:00.000000000 Z
+date: 2020-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake