rbbt-util 5.13.35 → 5.13.36

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 81458a4ac4aaf570ca085039bc9edef474939dcf
-  data.tar.gz: 0347d3cdb338145d6633126373eaad1430aaea79
+  metadata.gz: 48414e48036dde6708ce828f47fa10e1169cf26c
+  data.tar.gz: 818bb5d1cf90da015d716050c9283821435eecbf
 SHA512:
-  metadata.gz: f3eed434127735a837685ffd0ceb170c4e05988cdb02cab6626d63b1c7f4968f62ceff494bf50a18ad10baef182a9cea2b4f135b2fef21814d472aaf7b7ffd73
-  data.tar.gz: 6c212fb6f2cd6a28ff2e45a3fcceffccc571a02f802f45627c0b7299b2df38461eb8181434ac7a21caf83b9c1855a2d51d1fdb9f7a9b27501bcc5b84b3481232
+  metadata.gz: a35a5de8c4870b10e86353030787fb261b7086eac2553e79b86609352d88d69d09f65246cbd07e43f65d1a9bf863c0f00e070c9f380b4f2ae1d541b6f89f61a9
+  data.tar.gz: e649af70a02eb33d3a3255766bf495197baa48d0274fa3a6b8e723a43daa4f8dc497e7aa781291ad83d75b6a50fa4f2913a1530bb9924147ecd670c4ce672fc5

data/lib/rbbt/persist.rb

@@ -204,6 +204,7 @@ module Persist
     ConcurrentStream.setup(out, :threads => saver_thread, :filename => path)
     out.callback = callback
     out.abort_callback = abort_callback
+    out.lockfile = stream.lockfile
     out
   end
 

data/lib/rbbt/tsv/stream.rb

@@ -57,22 +57,22 @@ module TSV
     options = Misc.add_defaults options, :sep => "\t", :sort => true
     sort, sep = Misc.process_options options, :sort, :sep
 
-    Misc.open_pipe do |sin|
+    streams = streams.collect do |stream|
+      if defined? Step and Step === stream
+        stream.grace
+        stream.get_stream || stream.join.path.open
+      else
+        stream
+      end
+    end
+
+    out = Misc.open_pipe do |sin|
       num_streams = streams.length
 
-      streams = streams.collect do |stream|
-        if defined? Step and Step === stream
-          stream.grace
-          stream.get_stream || stream.join.path.open
-        else
-          stream
-        end
-      end
 
       streams = streams.collect do |stream|
         sorted = Misc.sort_stream(stream)
         stream.annotate sorted if stream.respond_to? :annotate
-        stream.clear if stream.respond_to? :annotate
         sorted
       end if sort
 
@@ -161,5 +161,7 @@ module TSV
         raise $!
       end
     end
+
+    out
   end
 end

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -7,13 +7,17 @@ class RbbtProcessQueue
 
       @pid = Process.fork do
         begin
-          Persist::CONNECTIONS.values.each do |db| db.close if db.write? end
-          ObjectSpace.each_object(Mutex) do |m| 
-            begin 
-              m.unlock 
-            rescue ThreadError
-            end if m.locked? 
-          end
+
+          #Persist::CONNECTIONS.values.each do |db| db.close if db.write? end
+          #ObjectSpace.each_object(Mutex) do |m| 
+          #  begin 
+          #    m.unlock 
+          #  rescue ThreadError
+          #  end if m.locked? 
+          #end
+          
+          Misc.pre_fork
+
           @cleanup.call if @cleanup
           @queue.close_write 
 

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -2,7 +2,7 @@ module ConcurrentStream
   attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile
 
   def self.setup(stream, options = {}, &block)
-    threads, pids, callback, filename, autojoin = Misc.process_options options, :threads, :pids, :callback, :filename, :autojoin
+    threads, pids, callback, filename, autojoin, lockfile = Misc.process_options options, :threads, :pids, :callback, :filename, :autojoin, :lockfile
     stream.extend ConcurrentStream unless ConcurrentStream === stream
 
     stream.threads ||= []
@@ -24,18 +24,13 @@ module ConcurrentStream
 
     stream.filename = filename unless filename.nil?
 
+    stream.lockfile = lockfile if lockfile
+
     stream
   end
 
   def annotate(stream)
-    ConcurrentStream.setup stream
-    stream.threads = threads
-    stream.pids = pids
-    stream.callback = callback
-    stream.abort_callback = abort_callback
-    stream.filename = filename
-    stream.autojoin = autojoin
-    stream.joined = joined
+    ConcurrentStream.setup(stream, :threads => threads, :pids => pids, :callback => callback, :abort_callback => abort_callback, :filename => filename, :autojoin => autojoin, :lockfile => lockfile)
     stream
   end
 
@@ -138,6 +133,8 @@ module ConcurrentStream
 
       abort_threads
       abort_pids
+      iii [:abort, lockfile]
+      lockfile.unlock if lockfile and lockfile.locked?
     end
     Log.medium "Aborted stream #{Misc.fingerprint self} -- #{@abort_callback} [#{@aborted}]"
   end

data/lib/rbbt/util/misc/development.rb

@@ -1,5 +1,15 @@
 module Misc
 
+  def self.pre_fork
+    Persist::CONNECTIONS.values.each do |db| db.close if db.write? end
+    ObjectSpace.each_object(Mutex) do |m| 
+      begin 
+        m.unlock 
+      rescue ThreadError
+      end if m.locked? 
+    end
+  end
+
   def self.string2const(string)
     return nil if string.nil?
     mod = Kernel

data/lib/rbbt/workflow.rb

@@ -238,7 +238,7 @@ module Workflow
               m = Module.new
 
               helpers.each do |name,block|
-                m.class.send(:define_method, name, &block)
+                m.send(:define_method, name, &block)
               end
 
               m

data/lib/rbbt/workflow/step/run.rb

@@ -92,7 +92,7 @@ class Step
       @result = self._exec
       @result = @result.stream if TSV::Dumper === @result
     end
-    no_load or ENV["RBBT_NO_STREAM"] ? @result : prepare_result(@result, @task.result_description)
+    (no_load or ENV["RBBT_NO_STREAM"]) ? @result : prepare_result(@result, @task.result_description)
   end
 
   def checks
@@ -273,6 +273,7 @@ class Step
   def fork(semaphore = nil)
     raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil? and not Process.pid == @pid and Misc.pid_exists?(@pid) and not done? and info[:forked]
     @pid = Process.fork do
+      Misc.pre_fork
       begin
         RbbtSemaphore.wait_semaphore(semaphore) if semaphore
         FileUtils.mkdir_p File.dirname(path) unless Open.exists? File.dirname(path)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.13.35
+  version: 5.13.36
 platform: ruby
 authors:
 - Miguel Vazquez