rbbt-util 5.11.2 → 5.11.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06c3a98f22b69a5738d6c9b478a2cee9a4e52e66
-  data.tar.gz: d60e07e6371c2c6c88359d6b028e7c7fdcbd0c64
+  metadata.gz: add772f9d207913045a9042098c44b2c5549215c
+  data.tar.gz: 5932efc5c957bc4cf93de0228d4e654a47a788ce
 SHA512:
-  metadata.gz: dab5cbee06c6169ab384555b53e6ea3407cb2c3f18b0b8d99f9aac6d6209d4c8656a6baeba4d46ede3c87c8eb141a870723d7a8336f7f58c85535d6d2f3f54a6
-  data.tar.gz: e3268aebe76ca88cfbc948b34367992188e38289fe966fc90ffc07609e2f6b355d751327008b691fbe4e94a1d7b9085e32fd0f73734ee538b041b9bd982486b0
+  metadata.gz: 864893d125d7ca6e03736829a5c77d11e936229dfd3569139c22b1acd10d8430156603b8a23f3ecf5c9b0602040278b6fff1482ccddde3cc9ca3686f40c16791
+  data.tar.gz: 71da4b77c4fad1c83fcaa068118150bd6cfbc0e6e029a2bd3ffc2867293fa585cba4a4638eff98df9d4120fe2a95847c290ae7f6b07bfad2699ba15566f1a99c

data/lib/rbbt/persist.rb

@@ -83,48 +83,53 @@ module Persist
 
   TRUE_STRINGS = Set.new ["true", "True", "TRUE", "t", "T", "1", "yes", "Yes", "YES", "y", "Y", "ON", "on"] unless defined? TRUE_STRINGS
   def self.load_file(path, type)
-    case (type || :marshal).to_sym
-    when :nil
-      nil
-    when :boolean
-      TRUE_STRINGS.include? Open.read(path).chomp.strip
-    when :annotations
-      Annotated.load_tsv TSV.open(path)
-    when :tsv
-      TSV.open(path)
-    when :marshal_tsv
-      TSV.setup(Marshal.load(Open.open(path)))
-    when :fwt
-      FixWidthTable.get(path) 
-    when :string, :text
-      Open.read(path)
-    when :binary
-      f = File.open(path, 'rb')
-      res = f.read
-      f.close
-      res.force_encoding("ASCII-8BIT") if res.respond_to? :force_encoding
-      res
-    when :array
-      res = Open.read(path).split("\n", -1)
-      res.pop
-      res
-    when :marshal
-      Marshal.load(Open.open(path))
-    when :yaml
-      YAML.load(Open.open(path))
-    when :float
-      Open.read(path).to_f
-    when :integer
-      Open.read(path).to_i
-    else
-      raise "Unknown persistence: #{ type }"
+    begin
+      case (type || :marshal).to_sym
+      when :nil
+        nil
+      when :boolean
+        TRUE_STRINGS.include? Open.read(path).chomp.strip
+      when :annotations
+        Annotated.load_tsv TSV.open(path)
+      when :tsv
+        TSV.open(path)
+      when :marshal_tsv
+        TSV.setup(Marshal.load(Open.open(path)))
+      when :fwt
+        FixWidthTable.get(path) 
+      when :string, :text
+        Open.read(path)
+      when :binary
+        f = File.open(path, 'rb')
+        res = f.read
+        f.close
+        res.force_encoding("ASCII-8BIT") if res.respond_to? :force_encoding
+        res
+      when :array
+        res = Open.read(path).split("\n", -1)
+        res.pop
+        res
+      when :marshal
+        Marshal.load(Open.open(path))
+      when :yaml
+        YAML.load(Open.open(path))
+      when :float
+        Open.read(path).to_f
+      when :integer
+        Open.read(path).to_i
+      else
+        raise "Unknown persistence: #{ type }"
+      end
+    rescue
+      Log.error "Exception loading #{ type } #{ path }: #{$!.message}"
+      raise $!
     end
   end
 
   def self.save_file(path, type, content)
 
     return if content.nil?
-    
+
     case (type || :marshal).to_sym
     when :nil
       nil
@@ -248,9 +253,11 @@ module Persist
             end
           rescue Aborted
             Log.error "Tee stream thread aborted"
-            stream.abort if stream.respond_to? :abort
+            sout.abort if sout.respond_to? :abort
+            sin.abort if sin.respond_to? :abort
           rescue Exception
-            stream.abort if stream.respond_to? :abort
+            sin.abort if sin.respond_to? :abort
+            sout.abort if sout.respond_to? :abort
             Log.exception $!
             parent.raise $!
           ensure
@@ -413,6 +420,7 @@ module Persist
                 res = tee_stream(res.stream, path, type, res.respond_to?(:callback)? res.callback : nil)
                 ConcurrentStream.setup res do
                   begin
+                    iii [:unlock, lockfile.path]
                     lockfile.unlock
                   rescue
                     Log.warn "Lockfile exception: " << $!.message

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -199,12 +199,9 @@ module TSV
         end
       end
 
-      #stream = obj_stream(obj)
-      #stream.close if stream
-
       Process.waitpid pid
     rescue Exception
-      Log.exception $!
+      Log.error "Exception traversin in cpus: #{$!.message}"
       q.abort
       raise $!
     ensure

data/lib/rbbt/util/concurrency/processes/socket.rb

@@ -9,7 +9,7 @@ class RbbtProcessQueue
     def initialize
       @sread, @swrite = Misc.pipe
 
-      key = rand(100000).to_s;
+      key = rand(100000000).to_s;
       @write_sem = key + '.in'
       @read_sem = key + '.out'
       RbbtSemaphore.create_semaphore(@write_sem,1)

data/lib/rbbt/util/misc.rb

@@ -35,7 +35,7 @@ module LaterString
 end
 
 module ConcurrentStream
-  attr_accessor :threads, :pids, :callback, :filename, :joined
+  attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined
 
   def joined?
     @joined
@@ -72,6 +72,7 @@ module ConcurrentStream
   def abort
     @threads.each{|t| t.raise Aborted.new } if @threads
     @pids.each{|pid| Process.kill :INT, pid } if @pids
+    @abort_callback.call if @abort_callback
   end
 
   def self.setup(stream, options = {}, &block)

data/lib/rbbt/workflow.rb

@@ -234,7 +234,8 @@ module Workflow
   def get_job_step(step_path, task = nil, input_values = nil, dependencies = nil)
     step_path = step_path.call if Proc === step_path
     persist = input_values.nil? ? false : true
-    step = Persist.memory("Step", :key => step_path, :repo => step_cache, :persist => persist ) do
+    key = Path === step_path ? step_path.find : step_path
+    step = Persist.memory("Step", :key => key, :repo => step_cache, :persist => persist ) do
       step = Step.new step_path, task, input_values, dependencies
 
       helpers.each do |name, block|

data/lib/rbbt/workflow/accessor.rb

@@ -105,10 +105,21 @@ class Step
   def self.log(status, message, path, &block)
     if block_given?
       start = Time.now
+      status = status.to_s
+      status_color = case status
+                     when "starting"
+                       :yellow
+                     when "error"
+                       :red
+                     when "done"
+                       :green
+                     else
+                       :cyan
+                     end
       Log.info do 
         now = Time.now
         str = Log.color :reset
-        str << "#{ Log.color :cyan, status.to_s }"
+        str << "#{ Log.color status_color, status}"
         str << ": #{ message }" if message
         str << " -- #{Log.color :blue, path.to_s}" if path
         str
@@ -123,10 +134,21 @@ class Step
       end
       res
     else
+      status = status.to_s
+      status_color = case status
+                     when "starting"
+                       :yellow
+                     when "error"
+                       :red
+                     when "done"
+                       :green
+                     else
+                       :cyan
+                     end
       Log.info do 
         now = Time.now
         str = Log.color :reset
-        str << "#{ Log.color :cyan, status.to_s }"
+        str << "#{ Log.color status_color, status}"
         str << ": #{ message }" if message
         str << " -- #{Log.color :blue, path.to_s}" if path
         str

data/lib/rbbt/workflow/step.rb

@@ -52,7 +52,7 @@ class Step
         def log(status, message = nil)
           self.status = status
           message message
-          relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message )
+          relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless relay_step.done? or relay_step.error? or relay_step.aborted?
         end
       end
     end
@@ -80,8 +80,8 @@ class Step
       rescue Exception
         value.abort if value.respond_to? :abort
       ensure
-        value.close unless value.closed?
         value.join if value.respond_to? :join
+        value.close unless value.closed?
       end
     when (not defined? Entity or description.nil? or not Entity.formats.include? description)
       value
@@ -152,10 +152,26 @@ class Step
     rec_dependencies.collect{|dependency| dependency.path }.uniq
   end
 
+  def kill_children
+    children_pids = info[:children_pids]
+    if children_pids and children_pids.any?
+      Log.medium("Killing children: #{ children_pids * ", " }")
+      children_pids.each do |pid|
+        Log.medium("Killing child #{ pid }")
+        begin
+          Process.kill "INT", pid
+        rescue Exception
+          Log.medium("Exception killing child #{ pid }: #{$!.message}")
+        end
+      end
+    end
+  end
+
   def run(no_load = false)
 
     @mutex.synchronize do
-      result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load ? :stream : false do
+      no_load = no_load ? :stream : false
+      result = Persist.persist "Job", @task.result_type, :file => path, :check => checks, :no_load => no_load do |lockfile|
         if Step === Step.log_relay_step and not self == Step.log_relay_step
           relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
         end
@@ -176,7 +192,8 @@ class Step
             dependency.relay_log self
             dependency.clean if not dependency.done? and dependency.error?
             dependency.clean if dependency.streaming? and not dependency.running?
-            dependency.run true unless dependency.result 
+            dependency.run true unless dependency.result or dependency.done?
+            seen_deps << dependency.path
             seen_deps.concat dependency.rec_dependencies.collect{|d| d.path} 
           rescue Exception
             backtrace = $!.backtrace
@@ -196,29 +213,30 @@ class Step
         rescue Aborted
           log(:error, "Aborted")
 
-          children_pids = info[:children_pids]
-          if children_pids and children_pids.any?
-            Log.medium("Killing children: #{ children_pids * ", " }")
-            children_pids.each do |pid|
-              Log.medium("Killing child #{ pid }")
-              begin
-                Process.kill "INT", pid
-              rescue Exception
-                Log.medium("Exception killing child #{ pid }: #{$!.message}")
-              end
-            end
-          end
-
+          #          children_pids = info[:children_pids]
+          #          if children_pids and children_pids.any?
+          #            Log.medium("Killing children: #{ children_pids * ", " }")
+          #            children_pids.each do |pid|
+          #              Log.medium("Killing child #{ pid }")
+          #              begin
+          #                Process.kill "INT", pid
+          #              rescue Exception
+          #                Log.medium("Exception killing child #{ pid }: #{$!.message}")
+          #              end
+          #            end
+          #          end
+          
+          kill_children
           raise $!
         rescue Exception
           backtrace = $!.backtrace
 
           # HACK: This fixes an strange behaviour in 1.9.3 where some
           # backtrace strings are coded in ASCII-8BIT
-          backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
-
+          kill_children
           set_info :backtrace, backtrace 
           log(:error, "#{$!.class}: #{$!.message}")
+          backtrace.each{|l| l.force_encoding("UTF-8")} if String.instance_methods.include? :force_encoding
           raise $!
         end
 
@@ -234,6 +252,13 @@ class Step
               Log.exception $!
             end
           end
+          result.abort_callback = Proc.new do
+            begin
+              log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}"
+            rescue
+              Log.exception $!
+            end
+          end
         when TSV::Dumper
           log :streaming, "#{Log.color :magenta, "Streaming task result TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
           ConcurrentStream.setup result.stream do
@@ -246,6 +271,13 @@ class Step
               Log.exception $!
             end
           end
+          result.stream.abort_callback = Proc.new do
+            begin
+              log :error, "#{Log.color :red, "ERROR -- streamming aborted"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}] -- #{path}"
+            rescue
+              Log.exception $!
+            end
+          end
         else
           set_info :done, (done_time = Time.now)
           set_info :time_elapsed, (time_elapsed = done_time - start_time)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.11.2
+  version: 5.11.3
 platform: ruby
 authors:
 - Miguel Vazquez