rbbt-util 5.20.26 → 5.21.0

data/lib/rbbt/tsv/util.rb

@@ -108,8 +108,13 @@ module TSV
         StringIO.new file
       end
     when (defined? Step and Step)
-      if file.respond_to?(:base_url) and file.result
-        file.result
+      if file.respond_to?(:base_url) 
+        if file.result and IO === file.result
+          file.result
+        else
+          file.join
+          get_stream(file.path, open_options.merge(:nocache => true))
+        end
       else
         file.grace
         stream = file.get_stream

data/lib/rbbt/util/log/progress/report.rb

@@ -105,6 +105,7 @@ module Log
 
     def save
       info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
+      info.delete_if{|k,v| v.nil?}
       Open.write(@file, info.to_yaml)
     end
 

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -104,7 +104,11 @@ module ConcurrentStream
 
   def join_callback
     if @callback and not joined?
-      @callback.call
+      begin
+        @callback.call
+      rescue Exception
+        Log.exception $!
+      end
       @callback = nil
     end
   end
@@ -162,7 +166,7 @@ module ConcurrentStream
 
   def abort(exception = nil)
     if @aborted
-      Log.medium "YET aborted stream #{Misc.fingerprint self} [#{@aborted}]"
+      Log.medium "Already aborted stream #{Misc.fingerprint self} [#{@aborted}]"
       return
     else
       Log.medium "Aborting stream #{Misc.fingerprint self} [#{@aborted}]"

data/lib/rbbt/util/misc/development.rb

@@ -1,20 +1,20 @@
 module Misc
 
-  def self.add_libdir(dir=nil)
+def self.add_libdir(dir=nil)
     dir ||= File.join(Path.caller_lib_dir(caller.first), 'lib')
     $LOAD_PATH.unshift(dir) unless $LOAD_PATH.include? dir
   end
 
   def self.pre_fork
-    Persist::CONNECTIONS.values.each do |db| 
-      db.close if db.write? 
+    Persist::CONNECTIONS.values.each do |db|
+      db.close if db.write?
     end
     Log::ProgressBar::BARS.clear
-    ObjectSpace.each_object(Mutex) do |m| 
-      begin 
-        m.unlock 
+    ObjectSpace.each_object(Mutex) do |m|
+      begin
+        m.unlock
       rescue ThreadError
-      end if m.locked? 
+      end if m.locked?
     end
   end
 
@@ -146,6 +146,8 @@ module Misc
     rescue TryAgain
       sleep sleep
       retry
+    rescue StopInsist
+      raise $!.exception
     rescue Aborted, Interrupt
       if msg
         Log.warn("Not Insisting after Aborted: #{$!.message} -- #{msg}")
@@ -158,7 +160,7 @@ module Misc
         Log.warn("Insisting after exception: #{$!.class} #{$!.message} -- #{msg}")
       else
         Log.warn("Insisting after exception:  #{$!.class} #{$!.message}")
-      end 
+      end
 
       if sleep and try > 0
         sleep sleep
@@ -216,14 +218,14 @@ module Misc
         else
           v, n = template[pos], template[-1]
           template.pop
-          template[pos] = n 
+          template[pos] = n
         end
         p << v
       end
     else
-      size.times do 
+      size.times do
         pos = nil
-        while pos.nil? 
+        while pos.nil?
           pos = (rand * total).floor
           if p.include? pos
             pos = nil
@@ -245,9 +247,11 @@ module Misc
     end
   end
 
+  MUTEX_FOR_THREAD_EXCLUSIVE = Mutex.new
+
   def self.object_delta(*args)
     res, delta = nil, nil
-    Thread.exclusive do
+    MUTEX_FOR_THREAD_EXCLUSIVE.synchronize do
       pre = Set.new
       delta = Set.new
 
@@ -264,7 +268,7 @@ module Misc
       end
 
     end
-    Log.info "Delta: #{delta.inspect}" 
+    Log.info "Delta: #{delta.inspect}"
     res
   end
 
@@ -362,7 +366,7 @@ module Misc
 
   PUSHBULLET_KEY=begin
                    if ENV["PUSHBULLET_KEY"]
-                     ENV["PUSHBULLET_KEY"] 
+                     ENV["PUSHBULLET_KEY"]
                    else
                      config_api = File.join(ENV['HOME'], 'config/apps/pushbullet/apikey')
                      if File.exist? config_api

data/lib/rbbt/util/misc/exceptions.rb

@@ -1,12 +1,13 @@
-class ParameterException < Exception; end
-class FieldNotFoundError < Exception;end
-class TryAgain < Exception; end
-class ClosedStream < Exception; end
+class RbbtException < StandardError; end
+class ParameterException < RbbtException; end
+class FieldNotFoundError < RbbtException;end
+class TryAgain < RbbtException; end
+class ClosedStream < RbbtException; end
 
-class ProcessFailed < Exception; end
-class Aborted < Exception; end
+class ProcessFailed < RbbtException; end
+class Aborted < RbbtException; end
 
-class RemoteServerError < Exception; end
+class RemoteServerError < RbbtException; end
 
 class DependencyError < Aborted
   def initialize(msg)
@@ -35,3 +36,11 @@ class KeepBar < Exception
     @payload = payload
   end
 end
+
+class StopInsist < Exception
+  attr_accessor :exception
+  def initialize(exception)
+    @exception = exception
+  end
+end
+

data/lib/rbbt/util/misc/inspect.rb

@@ -182,8 +182,9 @@ module Misc
         end
       end
 
-      if defined? Annotated and Annotated === _v and not (defined? AssociationItem and AssociationItem === _v)
-        info = Annotated.purge(_v.info)
+      if _v and defined? Annotated and Annotated === _v and not (defined? AssociationItem and AssociationItem === _v)
+        info = _v.info
+        info = Annotated.purge(info)
         str << "_" << hash2md5(info) 
       end
     end

data/lib/rbbt/util/misc/pipes.rb

@@ -23,12 +23,14 @@ module Misc
   OPEN_PIPE_IN = []
   def self.pipe
     OPEN_PIPE_IN.delete_if{|pipe| pipe.closed? }
-    PIPE_MUTEX.synchronize do
+    res = PIPE_MUTEX.synchronize do
       sout, sin = IO.pipe
       OPEN_PIPE_IN << sin
 
       [sout, sin]
     end
+    Log.debug{"Creating pipe #{[res.last.inspect,res.first.inspect] * " => "}"}
+    res
   end
   
   def self.release_pipes(*pipes)
@@ -188,6 +190,7 @@ module Misc
         end
 
         stream.join if stream.respond_to? :join
+
       rescue Aborted, Interrupt
         stream.abort if stream.respond_to? :abort
         out_pipes.each do |sout|
@@ -206,14 +209,14 @@ module Misc
     end
 
     out_pipes.each do |sout|
-      ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename
+      ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :_pair => stream
     end
 
-    abort_callback = Proc.new do
-      out_pipes.each do |s|
-        s.abort if s.respond_to? :abort
-      end
-    end
+    #abort_callback = Proc.new do
+    #  out_pipes.each do |s|
+    #    s.abort if s.respond_to? :abort
+    #  end
+    #end
 
     out_pipes
   end

data/lib/rbbt/util/task/job.rb

@@ -77,7 +77,7 @@ class Task
     end
 
     def info
-      return {} if not File.exists?(info_file)
+      return {} if not File.exist?(info_file)
       info = YAML.load(File.open(info_file)) || {}
       info.extend IndiferentHash
     end
@@ -146,7 +146,7 @@ class Task
     end
 
     def run_dependencies
-      required_files.each do |file| file.produce unless File.exists? file end unless required_files.nil?
+      required_files.each do |file| file.produce unless File.exist? file end unless required_files.nil?
       previous_jobs.each do |job| 
         if not job.recursive_done? 
           job.clean if job.error?
@@ -222,7 +222,7 @@ class Task
     def run
       return self if recursive_done?
       begin
-        FileUtils.rm info_file if File.exists? info_file
+        FileUtils.rm info_file if File.exist? info_file
         step(:started)
         start
         step(:done)
@@ -293,9 +293,9 @@ class Task
     end
 
     def clean
-      FileUtils.rm path if File.exists? path
-      FileUtils.rm info_file if File.exists? info_file
-      FileUtils.rm_rf path + '.files' if File.exists? path + '.files'
+      FileUtils.rm path if File.exist? path
+      FileUtils.rm info_file if File.exist? info_file
+      FileUtils.rm_rf path + '.files' if File.exist? path + '.files'
       self
     end
 

data/lib/rbbt/workflow.rb

@@ -181,7 +181,7 @@ module Workflow
   attr_accessor :libdir, :workdir 
   attr_accessor :helpers, :tasks
   attr_accessor :task_dependencies, :task_description, :last_task 
-  attr_accessor :asynchronous_exports, :synchronous_exports, :exec_exports
+  attr_accessor :stream_exports, :asynchronous_exports, :synchronous_exports, :exec_exports
   attr_accessor :step_cache
   attr_accessor :remote_tasks
 
@@ -237,6 +237,10 @@ module Workflow
     @task_description ||= {}
   end
 
+  def stream_exports
+    @stream_exports ||= []
+  end
+
   def asynchronous_exports
     @asynchronous_exports ||= []
   end
@@ -250,7 +254,7 @@ module Workflow
   end
   
   def all_exports
-    @all_exports ||= asynchronous_exports + synchronous_exports + exec_exports
+    @all_exports ||= asynchronous_exports + synchronous_exports + exec_exports + stream_exports
   end
 
   # {{{ JOB MANAGEMENT
@@ -404,4 +408,27 @@ module Workflow
       self.workdir = saved
     end
   end
+
+  def add_remote_tasks(remote_tasks)
+    remote_tasks.each do |remote, tasks|
+      tasks.each do |task|
+        self.remote_tasks[task.to_f] = remote
+      end
+    end
+  end
+
+  def self.process_remote_tasks(remote_tasks)
+    require 'rbbt/rest/client'
+    remote_tasks.each do |workflow, info|
+      wf = Workflow.require_workflow workflow
+      wf.remote_tasks ||= {}
+      info.each do |remote, tasks|
+        remote_wf = WorkflowRESTClient.new remote, workflow
+        tasks.each do |task|
+          wf.remote_tasks[task.to_sym] = remote_wf
+        end
+      end
+    end
+  end
+
 end

data/lib/rbbt/workflow/accessor.rb

@@ -315,6 +315,17 @@ class Step
     self._abort
   end
 
+  def recoverable_error?
+    return true if aborted?
+    return false unless error?
+    begin
+      klass = Kernel.const_get(info[:exception][:class])
+      not RbbtException === klass
+    rescue Exception
+      false
+    end
+  end
+
   def started?
     Open.exists?(path) or Open.exists?(pid_file)
   end
@@ -469,6 +480,8 @@ module Workflow
                :asynchronous
              when (exec_exports.include?(name.to_sym) or exec_exports.include?(name.to_s))
                :exec
+             when (stream_exports.include?(name.to_sym) or stream_exports.include?(name.to_s))
+               :stream
              else
                :none
              end
@@ -772,7 +785,7 @@ module Workflow
   end
 
   def task_exports
-    [exec_exports, synchronous_exports, asynchronous_exports].compact.flatten.uniq
+    [exec_exports, synchronous_exports, asynchronous_exports, stream_exports].compact.flatten.uniq
   end
 
 end

data/lib/rbbt/workflow/definition.rb

@@ -92,23 +92,42 @@ module Workflow
     tasks[name] = task
     task_dependencies[name] = consume_dependencies
   end
+
+  def unexport(*names)
+    names = names.collect{|n| n.to_s} + names.collect{|n| n.to_sym}
+    names.uniq!
+    exec_exports.replace exec_exports - names if exec_exports
+    synchronous_exports.replace synchronous_exports - names if synchronous_exports
+    asynchronous_exports.replace asynchronous_exports - names if asynchronous_exports
+    stream_exports.replace stream_exports - names if stream_exports
+  end
   
   def export_exec(*names)
+    unexport *names
     exec_exports.concat names
     exec_exports.uniq!
     exec_exports
   end
 
+  def export_synchronous(*names)
+    unexport *names
+    synchronous_exports.concat names
+    synchronous_exports.uniq!
+    synchronous_exports
+  end
+
   def export_asynchronous(*names)
+    unexport *names
     asynchronous_exports.concat names
     asynchronous_exports.uniq!
     asynchronous_exports
   end
 
-  def export_synchronous(*names)
-    synchronous_exports.concat names
-    synchronous_exports.uniq!
-    synchronous_exports
+  def export_stream(*names)
+    unexport *names
+    stream_exports.concat names
+    stream_exports.uniq!
+    stream_exports
   end
 
   alias export export_asynchronous

data/lib/rbbt/workflow/soap.rb

@@ -19,6 +19,7 @@ class WorkflowSOAP < SimpleWS
     @workflow = workflow
     @workflow.synchronous_exports.each do |name| synchronous name end
     @workflow.asynchronous_exports.each do |name| asynchronous name end
+    @workflow.stream_exports.each do |name| asynchronous name end
 
     desc "Job management: Check the status of a job"
     param_desc :jobid => "Job identifier", :return => "Status code. Special status codes are: 'done' and 'error'"

data/lib/rbbt/workflow/step/dependencies.rb

@@ -114,7 +114,7 @@ class Step
 
       if not dependency.started?
         log_dependency_exec(dependency, :starting)
-        dependency.run(:stream)
+        dependency.run(true)
         raise TryAgain
       end
 
@@ -154,6 +154,7 @@ class Step
       raise $!
     rescue Exception
       Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s } -- #{$!.message}"
+      Log.exception $!
       raise $!
     end
   end
@@ -187,7 +188,7 @@ class Step
         Misc.insist do
           begin
             step.produce
-          rescue
+          rescue Exception
             step.abort
             raise $!
           end
@@ -203,12 +204,13 @@ class Step
         Misc.insist do
           begin
             dep.produce 
+            Log.warn "Error producing #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
           rescue Aborted
             dep.abort
             raise $!
           rescue Exception
             dep.exception $!
-            raise $!
+            raise StopInsist.new($!)
           end
         end
         nil