rbbt-util 5.21.109 → 5.21.110

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 75212596904098b73851ec84788f599725a8cdf4
-  data.tar.gz: c353b207e28eaca9646f9102118c2bdeca997040
+  metadata.gz: 76533fd3b7702e342b7d237102246b920f687fb2
+  data.tar.gz: 2c2ab122805de8b07a901e001294b56f65c49eed
 SHA512:
-  metadata.gz: 6edbc878612e5727dbbd263e069f1e88330559b899ff90619c780fec660ce7ac1724f8f79722eb617158b479b3202ca28a116a8dd5cadbd517c016faed975356
-  data.tar.gz: 451774f390d7dbffa6a0f1ecd62b8cfac200129c31399f4ac9997297a58f2ee3c93192c16b15eee027349def00216f3c5c4cc5feef600f54b1f8b6c21b3501b1
+  metadata.gz: d2b46293054ceca952bff6da1f9c5200f278645911b55dd8b235b634ff0170f193fac20f5797d52d946f910eec894b44bae3a79b457c3c02cc8e0d011cd85f82
+  data.tar.gz: f3f15bb7057ea0521bdcbec6290cdde7e3dcd891abe2832008a6d17f944580a6a2fc284a9c442d421501be023c347f0003b01e1838911f363302c6059057c60e

data/lib/rbbt/rest/client/get.rb

@@ -81,8 +81,8 @@ class WorkflowRESTClient
     params = fix_params params
     res = capture_exception do
       Misc.insist(2, 0.5) do
-        Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
         raise "No url" if url.nil?
+        Log.debug{ "RestClient get_raw: #{ url } - #{Misc.fingerprint params}" }
         res = RestClient.get(self.encode(url), :params => params)
         raise TryAgain if res.code == 202
         res.to_s

data/lib/rbbt/rest/client/step.rb

@@ -130,12 +130,7 @@ class WorkflowRESTClient
       @done = @info && @info[:status] && @info[:status].to_sym == :done
       @info = Persist.memory("RemoteSteps Info", :url => @url, :persist => true, :update => !@done) do
         init_job unless @url
-        info = begin
-                 WorkflowRESTClient.get_json(File.join(@url, 'info'))
-               rescue
-                 clean
-                 raise $!
-               end
+        info = WorkflowRESTClient.get_json(File.join(@url, 'info'))
         info = WorkflowRESTClient.fix_hash(info)
         info[:status] = info[:status].to_sym if String === info[:status]
         info
@@ -150,13 +145,15 @@ class WorkflowRESTClient
         status = info[:status]
         @done = true if status and status.to_sym == :done
         status
+      rescue
+        nil
       ensure
         @info = nil
       end
     end
 
     def started?
-      @result != nil or @started or @streaming or @url
+      @result != nil || @started || @streaming
     end
 
     def done?
@@ -216,7 +213,7 @@ class WorkflowRESTClient
     end
 
     def running?
-      ! %w(done error aborted).include? status.to_s
+      ! %w(done error aborted noinfo).include? status.to_s
     end
 
     def path
@@ -361,8 +358,7 @@ class WorkflowRESTClient
       self
     end
 
-    def clean
-      init_job
+    def _clean
       begin
         params = {:_update => :clean}
         WorkflowRESTClient.clean_url(url, params) if @url
@@ -370,8 +366,17 @@ class WorkflowRESTClient
       rescue Exception
         Log.exception $!
       end
+    end
+
+    def clean
+      init_job
+      _clean
       self
     end
+
+    def input_checks
+      []
+    end
   end
 end
 

data/lib/rbbt/util/log.rb

@@ -13,8 +13,8 @@ module Log
 
 
   #ToDo: I'm not sure if using a Mutex here really gives troubles in CPU concurrency
-  #LOG_MUTEX = MockMutex.new
-  LOG_MUTEX = Mutex.new
+  LOG_MUTEX = MockMutex.new
+  #LOG_MUTEX = Mutex.new
 
   SEVERITY_NAMES ||= begin
                      names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR NONE ) 
@@ -122,7 +122,7 @@ module Log
   end
 
   def self.color(severity, str = nil, reset = false)
-    return str || "" if nocolor 
+    return str.dup || "" if nocolor 
     color = reset ? Term::ANSIColor.reset : ""
     color << SEVERITY_COLOR[severity] if Integer === severity
     color << Term::ANSIColor.send(severity) if Symbol === severity and Term::ANSIColor.respond_to? severity 

data/lib/rbbt/util/misc/pipes.rb

@@ -185,12 +185,7 @@ module Misc
         rescue IOError
         end
 
-        in_pipes.each do |sin|
-          sin.close unless sin.closed?
-        end
-
-        stream.join if stream.respond_to? :join
-
+        in_pipes.first.close
       rescue Aborted, Interrupt
         stream.abort if stream.respond_to? :abort
         out_pipes.each do |sout|
@@ -212,11 +207,12 @@ module Misc
       ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :_pair => stream
     end
 
-    #abort_callback = Proc.new do
-    #  out_pipes.each do |s|
-    #    s.abort if s.respond_to? :abort
-    #  end
-    #end
+    out_pipes.first.callback = Proc.new do 
+      stream.join
+      in_pipes[1..-1].each do |sin|
+        sin.close unless sin.closed?
+      end
+    end
 
     out_pipes
   end
@@ -387,12 +383,12 @@ module Misc
             raise $! unless File.exist? path
           end
 
+          FileUtils.touch path if File.exist? path
           content.join if content.respond_to? :join and not (content.respond_to?(:joined?) and content.joined?)
 
           if Lockfile === lock_options[:lock] and lock_options[:lock].locked?
             lock_options[:lock].unlock
           end
-          FileUtils.touch path if File.exist? path
           Open.notify_write(path) 
         rescue Aborted
           Log.medium "Aborted sensiblewrite -- #{ Log.reset << Log.color(:blue, path) }"

data/lib/rbbt/workflow/accessor.rb

@@ -358,7 +358,7 @@ class Step
   end
 
   def started?
-    Open.exists?(path) or Open.exists?(pid_file) #or Open.exists?(info_file)
+    Open.exists?(path) or (Open.exists?(pid_file) && Open.exists?(info_file))
   end
 
   def waiting?
@@ -366,9 +366,12 @@ class Step
   end
 
   def dirty?
+    return true if Open.exists?(pid_file) && ! ( Open.exists?(info_file) || done? )
+    return false unless done? || status == :done
+
     status = self.status
 
-    if done? and not status == :done and not status == :noinfo
+    if done? and not (status == :done or status == :ending) and not status == :noinfo
       return true 
     end
     if status == :done and not done?
@@ -376,7 +379,9 @@ class Step
     end
 
     dirty_files = rec_dependencies.reject{|dep|
-      (dep.path && Open.exists?(dep.path)) || (dep.error? && ! dep.recoverable_error?)
+      (defined?(WorkflowRESTClient) && WorkflowRESTClient::RemoteStep === dep) || 
+        (dep.path && (Open.exists?(dep.path) || Open.remote?(dep.path))) || 
+        (dep.error? && ! dep.recoverable_error?)
     }
 
     if dirty_files.any?

data/lib/rbbt/workflow/step/dependencies.rb

@@ -82,8 +82,8 @@ class Step
 
     if (status == 'error' && (job.recoverable_error? || job.dirty?)) ||
       job.aborted? ||
-      (job.done? && job.dirty?) ||
-      (status == 'waiting' && ! job.running?) 
+      (job.done? && job.dirty?)  ||
+      (! (job.status == :waiting || job.status == :noinfo || job.done? || job.error? || job.aborted?) && ! job.running?)
 
       job.clean 
     end
@@ -124,8 +124,7 @@ class Step
         return
       end
 
-      if dependency.aborted? or (dependency.error? and dependency.recoverable_error?) or dependency.missing?
-        log_dependency_exec(dependency, "aborted (clean)")
+      if dependency.aborted? or (dependency.error? and dependency.recoverable_error?) or (!Open.remote?(dependency.path) && dependency.missing?)
         dependency.clean
         raise TryAgain
       end
@@ -221,8 +220,8 @@ class Step
             end
           end
         end
-        nil
       end
+      nil
     when :bootstrap
       cpus = rest.nil? ? nil : rest.first 
       cpus = 5 if cpus.nil?
@@ -370,11 +369,20 @@ class Step
   def stop_dependencies
     return if dependencies.nil?
     dependencies.each do |dep|
+      if dep.nil?
+        Log.warn "Dependency is nil #{Misc.fingerprint step} -- #{Misc.fingerprint dependencies}"
+        next
+      end
+
       begin
         next if dep.done? or dep.aborted?
       rescue
       end
-      dep.abort
+
+      begin
+        dep.abort if dep.running?
+      rescue
+      end
     end
     kill_children
   end

data/lib/rbbt/workflow/step/run.rb

@@ -88,13 +88,12 @@ class Step
   def dependency_checks
     rec_dependencies.
       select{|dependency| ! (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency) }.
-      select{|dependency| ! dependency.error? }.
-      collect{|dependency| dependency.path }.uniq
+      select{|dependency| ! Open.remote?(dependency.path) }.
+      select{|dependency| ! dependency.error? }
   end
 
   def input_checks
-    inputs.select{|i| Step === i && i.done?}.
-      collect{|dependency| dependency.path }.uniq
+    inputs.select{|i| Step === i && i.done? }
   end
 
   def checks
@@ -102,7 +101,20 @@ class Step
   end
 
   def updated?
-    done? and checks.select{|path| File.mtime(path) > File.mtime(self.path)  }.empty?
+    return true unless done?
+    outdated_time  = []
+    outdated_dep  = []
+    checks.each do |dep| 
+      if File.mtime(dep.path) > File.mtime(self.path) 
+        outdated_time << dep
+      end
+      if ! dep.updated?  
+        outdated_dep << dep
+      end
+    end
+
+    outdated = outdated_time + outdated_dep
+    outdated.empty?
   end
 
   def kill_children
@@ -133,7 +145,7 @@ class Step
         no_load = :stream if no_load
 
         Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
-        result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do 
+        result = Persist.persist "Job", @task.result_type, :file => path, :check => checks.collect{|dep| dep.path}, :no_load => no_load do 
           if Step === Step.log_relay_step and not self == Step.log_relay_step
             relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
           end
@@ -152,10 +164,10 @@ class Step
             :workflow => (@workflow || @task.workflow).to_s,
             :task_name => @task.name,
             :result_type => @task.result_type,
-            :result_description => @task.result_description
+            :result_description => @task.result_description,
+            :dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
           })
 
-          set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
 
           begin
             run_dependencies
@@ -270,9 +282,11 @@ class Step
                 status = self.status
                 if status != :done and status != :error and status != :aborted
                   Misc.insist do
-                    set_info :done, (done_time = Time.now)
-                    set_info :total_time_elapsed, (total_time_elapsed = done_time - issue_time)
-                    set_info :time_elapsed, (time_elapsed = done_time - start_time)
+                    merge_info({
+                     :done => (done_time = Time.now),
+                     :total_time_elapsed => (total_time_elapsed = done_time - issue_time),
+                     :time_elapsed => (time_elapsed = done_time - start_time)
+                    })
                     log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
                   end
                 end
@@ -310,9 +324,11 @@ class Step
               raise exception
             end
           else
-            set_info :done, (done_time = Time.now)
-            set_info :total_time_elapsed, (total_time_elapsed = done_time - issue_time)
-            set_info :time_elapsed, (time_elapsed = done_time - start_time)
+            merge_info({
+              :done => (done_time = Time.now),
+              :total_time_elapsed => (total_time_elapsed = done_time - issue_time),
+              :time_elapsed => (time_elapsed = done_time - start_time)
+            })
             log :ending
             Step.purge_stream_cache
             FileUtils.rm pid_file if File.exist?(pid_file)
@@ -595,7 +611,7 @@ class Step
         dependencies.each{|dep| dep.join }
       end
 
-      until path.exists? or error? or aborted?
+      until (path.exists? && status == :done) or error? or aborted?
         sleep 1 
         join_stream if streaming?
       end

data/share/rbbt_commands/system/status

@@ -17,6 +17,7 @@ $ rbbt system status <workflow> <task>
 -q--quick Quick check
 -a--all Print all jobs, not only uncompleted
 -i--inputs* List of inputs to print
+-if--info_fields* List of info fields to print
 EOF
 rbbt_usage and exit 0 if options[:help]
 
@@ -29,6 +30,7 @@ task = task.split "," if task
 all = options.delete :all
 
 inputs = (options[:inputs] || "").split(",")
+info_fields = (options[:info_fields] || "").split(",")
 
 def pid_msg(pid)
   color = if pid and Misc.pid_exists? pid
@@ -145,6 +147,7 @@ workflows.sort.each do |workflow,tasks|
                rescue
                  {:status => :noinfo}
                end
+        IndiferentHash.setup(info)
 
         pid = info[:pid]
         status = info[:status]
@@ -163,15 +166,22 @@ workflows.sort.each do |workflow,tasks|
           job_inputs = info[:inputs]
           IndiferentHash.setup(job_inputs)
 
-          iii inputs
           inputs.each do |input|
             value = job_inputs[input]
-            iii [input, value]
             next if  value.nil?
             value_str = Misc.fingerprint(value)
             str << " #{input}=#{value_str}"
           end
         end
+
+        if info_fields and info_fields.any?
+          info_fields.each do |field|
+            value = info[field]
+            next if  value.nil?
+            value_str = Misc.fingerprint(value)
+            str << " #{field}=#{value_str}"
+          end
+        end
       end
       str << "; #{pid_msg pid}" unless status == "done"
       str

data/test/rbbt/test_workflow.rb

@@ -83,6 +83,11 @@ Returns numer * 2 lines containing TEST
     end
   end
 
+  dep :stream
+  task :stream2 => :array do
+    TSV.get_stream step(:stream)
+  end
+
   input :name, :string, "Name"
   task :input_dep => :text do |name|
     <<-EOF
@@ -207,5 +212,15 @@ class TestWorkflow < Test::Unit::TestCase
     nil
   end
 
+  def test_stream_order
+
+    Log.with_severity 0 do
+      job = TestWF.job(:stream2)
+      job.recursive_clean
+      job.produce
+
+    end
+  end
+
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.109
+  version: 5.21.110
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-23 00:00:00.000000000 Z
+date: 2017-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake