rbbt-util 5.21.5 → 5.21.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 641642d8cd4871a267b75c1d4c9e41eb0ff5c81b
-  data.tar.gz: 28b2ec23f29f39f04c618277058d7a691c17bc4c
+  metadata.gz: '08fc0e36be76ddc12da08a71a3866246380800e5'
+  data.tar.gz: 7e146e65ad2c37c8563b9d09b4d05f64f2cf467a
 SHA512:
-  metadata.gz: 28bb8d81971d76843af4e9f38b20a31bb66c5bead0b6ecd7967711207ac67d69e723bf14d51e2dbfc454804e073d614f89bbaae5026008facdd93d4352b9cc42
-  data.tar.gz: 88998bb05a0aa162e8cf6fb10a22fec38f4c78a8afaa83a2277d91060315346d55e46ce298e98c24862786e7e85020372940d7a9310f8a92822f670ee788cbbf
+  metadata.gz: 83141c8792d440fd891dd6a8ced6328cf92bd253545e8722c8cfc622a46071095b13a6eed0ce163f863aa5c8f09a3d1ba5e82c47b33f594dd5b692f3a58f6a6a
+  data.tar.gz: b2e711647c50de9ea09686872a35f23be5cb82fe1801b6b97942d2ad0cab0a17aa96dbf2e65d4f836d860438c6b892217a01e0b669bb2afd7f2848c6880ecd9e

data/lib/rbbt/entity/identifiers.rb

@@ -11,8 +11,6 @@ module Entity
       base.annotation :format
       base.annotation :organism
 
-
-
       class << base
         attr_accessor :identifier_files, :formats, :default_format, :name_format, :description_format
       end

data/lib/rbbt/rest/client/get.rb

@@ -53,6 +53,20 @@ class WorkflowRESTClient
     new_params
   end
 
+  def self.clean_url(url, params = {})
+    params = params.merge({ :_format => 'json', :update => 'clean' })
+    params = fix_params params
+    res = capture_exception do
+      Misc.insist(2, 0.5) do
+        Log.debug{ "RestClient clean: #{ url } - #{Misc.fingerprint params}" }
+        res = RestClient.get(URI.encode(url), :params => params)
+        raise TryAgain if res.code == 202
+        res
+      end
+    end
+    res
+  end
+ 
   def self.get_raw(url, params = {})
     params = params.merge({ :_format => 'raw' })
     params = fix_params params

data/lib/rbbt/rest/client/step.rb

@@ -38,12 +38,6 @@ class WorkflowRESTClient
                         exec(no_load)
                       elsif no_load == :stream
                         _run_job(:stream)
-                        #init_job 
-                        #join
-                        #Misc.open_pipe do |sin|
-                        #  body = get.body
-                        #  sin.write body
-                        #end
                       elsif no_load
                         init_job 
                         nil
@@ -95,7 +89,20 @@ class WorkflowRESTClient
     end
 
     def abort
+      return self if status == :done
       WorkflowRESTClient.get_json(@url + '?_update=abort') if @url and @name
+      self
+    end
+
+    def dup_inputs
+      return if @dupped or ENV["RBBT_NO_STREAM"] == 'true'
+      Log.low "Dupping inputs for #{path}"
+      dupped_inputs = {}
+      @inputs.collect do |k,input|
+        dupped_inputs[k] = Step.dup_stream input
+      end
+      @inputs = dupped_inputs
+      @dupped = true
     end
 
     def name
@@ -113,12 +120,20 @@ class WorkflowRESTClient
       init_job
       (Array === @url ? @url.first : @url).split("/")[-2]
     end
+    
+    def nopid?
+      false
+    end
 
     def info(check_lock=false)
       @done = @info && @info[:status] && @info[:status].to_sym == :done
       @info = Persist.memory("RemoteSteps Info", :url => @url, :persist => true, :update => !@done) do
         init_job unless @url
-        info = WorkflowRESTClient.get_json(File.join(@url, 'info'))
+        info = begin
+                 WorkflowRESTClient.get_json(File.join(@url, 'info'))
+               rescue
+                 {}
+               end
         info = WorkflowRESTClient.fix_hash(info)
         info[:status] = info[:status].to_sym if String === info[:status]
         info
@@ -185,7 +200,7 @@ class WorkflowRESTClient
         WorkflowRESTClient.post_jobname(File.join(base_url, task.to_s), inputs.merge(other_params).merge(:jobname => @name||@base_name, :_cache_type => cache_type))
       end
       @url = File.join(base_url, task.to_s, @name)
-      nil
+      self
     end
 
 
@@ -201,7 +216,7 @@ class WorkflowRESTClient
       if @url
         @url + '?_format=raw'
       else
-        [base_url, task, Misc.fingerprint(inputs)] * "/"
+        [base_url, task, @base_name + '-' +  Misc.fingerprint(inputs)] * "/"
       end
     end
 
@@ -225,12 +240,12 @@ class WorkflowRESTClient
         Misc.consume_stream(res, true) 
       end
 
-      if not self.done?
+      if not (self.done? || self.aborted? || self.error?)
         self.info 
-        return self if self.done?
-        sleep 0.2 unless self.done?
-        sleep 1 unless self.done?
-        while not self.done?
+        return self if self.done? || self.aborted? || self.error?
+        sleep 0.2 unless self.done? || self.aborted? || self.error?
+        sleep 1 unless self.done? || self.aborted? || self.error?
+        while not (self.done? || self.aborted? || self.error?)
           sleep 3
         end
       end
@@ -283,6 +298,7 @@ class WorkflowRESTClient
     def load
       params = {}
       join unless done? or streaming?
+      raise get_exception if error? or aborted?
       load_res get
     end
 
@@ -295,6 +311,7 @@ class WorkflowRESTClient
       @done = nil
       @name = nil
       @started = nil
+      @aborted = nil
       new_inputs = {}
       inputs.each do |k,i| 
         if File === i 
@@ -320,11 +337,9 @@ class WorkflowRESTClient
     end
 
     def clean
-      return
       begin
         params = {:_update => :clean}
-        init_job(nil, params)
-        WorkflowRESTClient.get_raw(url, params)
+        WorkflowRESTClient.clean_url(url, params) if @url
         _restart
       rescue Exception
         Log.exception $!

data/lib/rbbt/tsv/util.rb

@@ -122,6 +122,8 @@ module TSV
           stream
         else
           file.join
+          raise "Aborted stream from Step #{file.path}" if file.aborted?
+          raise "Exception in stream from Step #{file.path}: #{file.messages.last}" if file.error?
           get_stream(file.path)
         end
       end

data/lib/rbbt/workflow/accessor.rb

@@ -141,7 +141,7 @@ class Step
   def init_info
     return nil if @exec or info_file.nil?
     Open.lock(info_file, :lock => info_lock) do
-      i = {:status => :init}
+      i = {:status => :init, :pid => Process.pid}
       @info_cache = i
       Misc.sensiblewrite(info_file, INFO_SERIALIAZER.dump(i), :force => true, :lock => false)
       @info_cache_time = Time.now
@@ -315,6 +315,25 @@ class Step
     self._abort
   end
 
+  def get_exception
+    if info[:exception].nil?
+      raise Aborted if aborted?
+      raise messages.last if error?
+      return false 
+    else
+      ex_class, ex_message, ex_backtrace = info[:exception].values_at :class, :message, :backtrace
+      begin
+        klass = Kernel.const_get(ex_class)
+        ex = klass.new ex_message
+        #ex.set_backtrace ex_backtrace
+        ex
+      rescue
+        Log.exception $!
+        raise ex_message
+      end
+    end
+  end
+
   def recoverable_error?
     return true if aborted?
     return false unless error?
@@ -327,7 +346,7 @@ class Step
   end
 
   def started?
-    Open.exists?(path) or Open.exists?(pid_file)
+    Open.exists?(path) or Open.exists?(pid_file) or Open.exists?(info_file)
   end
 
   def dirty?
@@ -342,11 +361,11 @@ class Step
     IO === @result or @saved_stream or status == :streaming
   end
 
+
   def running?
-    return nil if not Open.exists? pid_file
-    return nil if info[:pid].nil?
+    pid = info[:pid]
+    return nil if pid.nil?
 
-    pid = @pid || info[:pid]
     if Misc.pid_exists?(pid) 
       pid
     else
@@ -358,8 +377,13 @@ class Step
     status == :error
   end
 
+  def nopid?
+    pid = info[:pid]
+    pid.nil? && ! (status == :aborted || status == :done || status == :error)
+  end
+
   def aborted?
-    @aborted || status == :aborted
+    status == :aborted || nopid?
   end
 
   # {{{ INFO

data/lib/rbbt/workflow/step/dependencies.rb

@@ -73,15 +73,20 @@ class Step
   end
 
   def self.prepare_for_execution(job)
-    return if (job.done? and not job.dirty?) or 
-    (job.streaming? and job.running?) or 
-    (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === job and not (job.error? or job.aborted?))
+    return if job.done? && ! job.dirty?
 
-    if job.error? or job.aborted? or (job.started? and not job.running? and not job.error?)
+    status = job.status.to_s
+    if defined? WorkflowRESTClient && WorkflowRESTClient::RemoteStep === job 
+      return if ! (status == 'done' and status == 'error' and status == 'aborted')
+    else
+      return if status == 'streaming' && job.running?
+    end
+
+    if (status == 'error' || job.aborted?) && job.recoverable_error?
       job.clean 
     end
 
-    job.dup_inputs unless job.done? or job.started?
+    job.dup_inputs unless status == 'done' or job.started?
 
     raise DependencyError, job if job.error?
   end
@@ -208,9 +213,10 @@ class Step
         Misc.insist do
           begin
             dep.produce 
-            Log.warn "Error producing #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
+            Log.warn "Error in bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
           rescue Aborted
             dep.abort
+            Log.warn "Aborted bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
             raise $!
           rescue Exception
             dep.exception $!
@@ -323,8 +329,13 @@ class Step
   end
 
   def stop_dependencies
+    return if dependencies.nil?
     dependencies.each do |dep|
-      dep.abort unless dep.done? or dep.aborted?
+      begin
+        next if dep.done? or dep.aborted?
+      rescue
+      end
+      dep.abort
     end
     kill_children
   end

data/lib/rbbt/workflow/step/run.rb

@@ -107,7 +107,6 @@ class Step
           log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task.name.to_s || ""}"
 
           merge_info({
-            :pid => Process.pid,
             :issued => (issue_time = Time.now),
             :name => name,
             :clean_name => clean_name,
@@ -361,7 +360,6 @@ class Step
       begin
         Log.medium "Aborting job stream #{stream.inspect} -- #{Log.color :blue, path}"
         stream.abort 
-        #stream.close unless stream.closed?
       rescue Aborted, Interrupt
         Log.medium "Aborting job stream #{stream.inspect} ABORTED RETRY -- #{Log.color :blue, path}"
         Log.exception $!
@@ -371,7 +369,7 @@ class Step
   end
 
   def _clean_finished
-    if Open.exists? path
+    if Open.exists? path and not status == :done
       Log.warn "Aborted job had finished. Removing result -- #{ path }"
       begin
         Open.rm path
@@ -384,12 +382,12 @@ class Step
   def _abort
     return if @aborted
     @aborted = true
-    return if done?
     Log.medium{"#{Log.color :red, "Aborting"} #{Log.color :blue, path}"}
     begin
+      return if done?
       stop_dependencies
       abort_stream
-      abort_pid if defined? @forked and @forked and running?
+      abort_pid if running?
     rescue Aborted, Interrupt
       Log.medium{"#{Log.color :red, "Aborting ABORTED RETRY"} #{Log.color :blue, path}"}
       retry
@@ -402,6 +400,7 @@ class Step
   end
 
   def abort
+    return if done? and status == :done
     _abort
     log(:aborted, "Job aborted") unless aborted? or error?
     self

data/share/config.ru

@@ -65,6 +65,9 @@ load_file Rbbt.etc['app.d/post.rb'].find_all
 #{{{ PRELOAD
 load_file Rbbt.etc['app.d/preload.rb'].find_all
 
+#{{{ SINATRA
+load_file Rbbt.lib['sinatra.rb'].find_all
+
 #{{{ RUN
 $title = app_name
 require 'rack'

data/share/rbbt_commands/system/clean

@@ -88,11 +88,12 @@ jobs.each do |file,i|
 
   status = info[:status].to_s
   status = :missing if status == "done" and not File.exist? file
+  status = :nopid if status != "done" and pid.nil?
   status = :dead if status != "done" and pid and not Misc.pid_exists?(pid)
   status = :sync if status != "done" and File.exist? file
 
   status = status.to_s
-  next unless status =~ /\berror$/ or status =~ /\bmissing$/ or status =~ /\baborted$/ or status =~ /\bdead$/ or status =~ /\bsync$/ or status == "" or (force and status == 'noinfo')
+  next unless status =~ /\bnopid/ or status =~ /\berror$/ or status =~ /\bmissing$/ or status =~ /\baborted$/ or status =~ /\bdead$/ or status =~ /\bsync$/ or status == "" or (force and status == 'noinfo')
   puts "  Removing #{ file } - #{status}" 
   Step.clean(file) 
 end

data/share/rbbt_commands/workflow/info

@@ -71,7 +71,7 @@ exception = info[:exception]
 rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :pid]
 
 puts Log.color(:magenta, "File") << ": " << step.path
-puts Log.color(:magenta, "Status") << ": " << status_msg(status)
+puts Log.color(:magenta, "Status") << ": " << status_msg(status) << ((step.aborted? || step.error?) && step.recoverable_error? ? " (recoverable)" : "" )
 puts Log.color(:magenta, "Pid") << ": " << pid_msg(pid, status.to_s == "done")
 puts Log.color(:magenta, "Time") << ": " << time.to_i.to_s << " sec." if time
 puts Log.color(:magenta, "Total time") << ": " << total_time.to_i.to_s << " sec." if time

data/share/rbbt_commands/workflow/prov

@@ -62,7 +62,7 @@ def report(step, offset = 0)
   info = step.info || {}
   path  = step.path
   status = info[:status] || :missing
-  status = :remote if Open.remote?(path)
+  status = "remote" if Open.remote?(path)
   name = info[:name] || File.basename(path)
   status = :unsync if status == :done and not File.exist? path
   str = " " * offset

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.5
+  version: 5.21.6
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-01 00:00:00.000000000 Z
+date: 2016-08-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -484,82 +484,82 @@ signing_key:
 specification_version: 4
 summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
 test_files:
-- test/rbbt/test_workflow.rb
+- test/test_helper.rb
 - test/rbbt/resource/test_path.rb
-- test/rbbt/util/test_cmd.rb
-- test/rbbt/util/simpleopt/test_setup.rb
-- test/rbbt/util/simpleopt/test_get.rb
-- test/rbbt/util/simpleopt/test_parse.rb
-- test/rbbt/util/test_chain_methods.rb
-- test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/association/test_item.rb
+- test/rbbt/association/test_database.rb
+- test/rbbt/association/test_open.rb
+- test/rbbt/association/test_index.rb
+- test/rbbt/association/test_util.rb
+- test/rbbt/util/test_concurrency.rb
 - test/rbbt/util/test_log.rb
+- test/rbbt/util/test_chain_methods.rb
+- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/simpleopt/test_parse.rb
+- test/rbbt/util/simpleopt/test_get.rb
+- test/rbbt/util/simpleopt/test_setup.rb
+- test/rbbt/util/test_cmd.rb
+- test/rbbt/util/test_semaphore.rb
+- test/rbbt/util/concurrency/test_threads.rb
+- test/rbbt/util/concurrency/processes/test_socket.rb
+- test/rbbt/util/concurrency/test_processes.rb
+- test/rbbt/util/test_tmpfile.rb
 - test/rbbt/util/test_open.rb
+- test/rbbt/util/test_filecache.rb
+- test/rbbt/util/R/test_eval.rb
+- test/rbbt/util/R/test_model.rb
+- test/rbbt/util/test_simpleDSL.rb
+- test/rbbt/util/log/test_progress.rb
+- test/rbbt/util/test_colorize.rb
+- test/rbbt/util/test_R.rb
 - test/rbbt/util/misc/test_lock.rb
-- test/rbbt/util/misc/test_multipart_payload.rb
-- test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/misc/test_pipes.rb
+- test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/misc/test_omics.rb
-- test/rbbt/util/test_concurrency.rb
-- test/rbbt/util/test_R.rb
-- test/rbbt/util/log/test_progress.rb
-- test/rbbt/util/test_colorize.rb
-- test/rbbt/util/test_simpleopt.rb
+- test/rbbt/util/misc/test_multipart_payload.rb
 - test/rbbt/util/test_excel2tsv.rb
-- test/rbbt/util/test_filecache.rb
-- test/rbbt/util/concurrency/test_processes.rb
-- test/rbbt/util/concurrency/test_threads.rb
-- test/rbbt/util/concurrency/processes/test_socket.rb
-- test/rbbt/util/test_semaphore.rb
 - test/rbbt/util/test_misc.rb
-- test/rbbt/util/test_tmpfile.rb
-- test/rbbt/util/R/test_model.rb
-- test/rbbt/util/R/test_eval.rb
-- test/rbbt/test_packed_index.rb
-- test/rbbt/entity/test_identifiers.rb
-- test/rbbt/test_association.rb
-- test/rbbt/knowledge_base/test_traverse.rb
-- test/rbbt/knowledge_base/test_registry.rb
-- test/rbbt/knowledge_base/test_entity.rb
-- test/rbbt/knowledge_base/test_enrichment.rb
-- test/rbbt/knowledge_base/test_syndicate.rb
-- test/rbbt/knowledge_base/test_query.rb
-- test/rbbt/test_resource.rb
 - test/rbbt/test_entity.rb
-- test/rbbt/test_knowledge_base.rb
-- test/rbbt/annotations/test_util.rb
-- test/rbbt/association/test_index.rb
-- test/rbbt/association/test_item.rb
-- test/rbbt/association/test_open.rb
-- test/rbbt/association/test_util.rb
-- test/rbbt/association/test_database.rb
-- test/rbbt/test_tsv.rb
 - test/rbbt/workflow/step/test_dependencies.rb
-- test/rbbt/workflow/test_task.rb
-- test/rbbt/workflow/test_step.rb
 - test/rbbt/workflow/test_doc.rb
-- test/rbbt/test_monitor.rb
-- test/rbbt/test_persist.rb
-- test/rbbt/test_annotations.rb
-- test/rbbt/persist/test_tsv.rb
-- test/rbbt/persist/tsv/test_lmdb.rb
-- test/rbbt/persist/tsv/test_kyotocabinet.rb
-- test/rbbt/persist/tsv/test_sharder.rb
-- test/rbbt/persist/tsv/test_cdb.rb
-- test/rbbt/persist/tsv/test_tokyocabinet.rb
-- test/rbbt/persist/tsv/test_leveldb.rb
-- test/rbbt/tsv/test_field_index.rb
+- test/rbbt/workflow/test_step.rb
+- test/rbbt/workflow/test_task.rb
+- test/rbbt/test_association.rb
+- test/rbbt/test_knowledge_base.rb
+- test/rbbt/tsv/parallel/test_traverse.rb
+- test/rbbt/tsv/parallel/test_through.rb
 - test/rbbt/tsv/test_parallel.rb
-- test/rbbt/tsv/test_index.rb
-- test/rbbt/tsv/test_matrix.rb
+- test/rbbt/tsv/test_accessor.rb
 - test/rbbt/tsv/test_change_id.rb
-- test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_stream.rb
-- test/rbbt/tsv/test_util.rb
-- test/rbbt/tsv/test_accessor.rb
 - test/rbbt/tsv/test_filter.rb
+- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_attach.rb
 - test/rbbt/tsv/test_manipulate.rb
-- test/rbbt/tsv/parallel/test_through.rb
-- test/rbbt/tsv/parallel/test_traverse.rb
+- test/rbbt/tsv/test_field_index.rb
+- test/rbbt/tsv/test_index.rb
+- test/rbbt/tsv/test_util.rb
+- test/rbbt/tsv/test_parser.rb
+- test/rbbt/test_packed_index.rb
+- test/rbbt/test_persist.rb
 - test/rbbt/test_fix_width_table.rb
-- test/test_helper.rb
+- test/rbbt/knowledge_base/test_traverse.rb
+- test/rbbt/knowledge_base/test_entity.rb
+- test/rbbt/knowledge_base/test_query.rb
+- test/rbbt/knowledge_base/test_enrichment.rb
+- test/rbbt/knowledge_base/test_syndicate.rb
+- test/rbbt/knowledge_base/test_registry.rb
+- test/rbbt/entity/test_identifiers.rb
+- test/rbbt/test_monitor.rb
+- test/rbbt/test_workflow.rb
+- test/rbbt/test_annotations.rb
+- test/rbbt/annotations/test_util.rb
+- test/rbbt/test_resource.rb
+- test/rbbt/persist/tsv/test_tokyocabinet.rb
+- test/rbbt/persist/tsv/test_kyotocabinet.rb
+- test/rbbt/persist/tsv/test_lmdb.rb
+- test/rbbt/persist/tsv/test_leveldb.rb
+- test/rbbt/persist/tsv/test_cdb.rb
+- test/rbbt/persist/tsv/test_sharder.rb
+- test/rbbt/persist/test_tsv.rb
+- test/rbbt/test_tsv.rb