rbbt-util 5.21.129 → 5.21.130

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f5f684d18bdb89966ce14703094359d62b64708d
-  data.tar.gz: 7298dc99105519116914c348280ef3a2f751ef5b
+  metadata.gz: 6996eb266a74a60fe3d8dbe789eccdc1aef28f63
+  data.tar.gz: cae5ae771a6fb59164c08952b189d399343e2be7
 SHA512:
-  metadata.gz: 9ddbbf8a627aca9d41640c4bda71fd44f3a29366cd3c0d015b23ac7c1e86b1ea964c04fc19719a10732f3fadd2506e2118dfea9aa55e782c820e595d9f639b40
-  data.tar.gz: 78800e637163f3ddfc6cee51ea06a19a58d9e47baa87ec73327268414c72c731236f49dd1f6d0a18b25eebaf1df6f4c7063ee52fbb626441ab41c55eb24e990e
+  metadata.gz: 60e9b3df6233872a4cbabd811a136d0778e7ba7a57833539c802ca7a7a7d8b94d3593fa35f5523d38d008e1327b579ad39cb96b11fcb94682fc569e529d56c51
+  data.tar.gz: b4aaa61f8fe810af9f0a6bdcf45b95bd6903563ffe21421f7987a7701f03af6c3fcb2dcb8be5c62ebc6ea842502229f9577bb2c2f5dbf87f01818fbb71854b21

data/lib/rbbt/association/index.rb

@@ -6,6 +6,7 @@ module Association
   def self.index(file, options = nil, persist_options = nil)
     options = options.nil? ? {} : options.dup
     persist_options = persist_options.nil? ?  Misc.pull_keys(options, :persist)  : persist_options.dup 
+    persist_options[:serializer] ||= options[:serializer] if options.include?(:serializer)
 
     persist_options = Misc.add_defaults persist_options.dup, :persist => true, :dir => Rbbt.var.associations
     persist = persist_options[:persist]
@@ -16,6 +17,8 @@ module Association
       recycle = options[:recycle]
       undirected = options[:undirected]
 
+      serializer = persist_options[:serializer] || :list
+
       persist_options[:file] = persist_options[:file] + '.database' if persist_options[:file]
 
       database = open(file, options, persist_options.dup.merge(:engine => "HDB"))
@@ -29,12 +32,12 @@ module Association
 
       key_field = [source_field, target_field, undirected ? "undirected" : nil].compact * "~"
 
-      TSV.setup(data, :key_field => key_field, :fields => fields[1..-1], :type => :list, :serializer => :list, :namespace => database.namespace)
+      TSV.setup(data, :key_field => key_field, :fields => fields[1..-1], :type => :list, :serializer => serializer, :namespace => database.namespace)
 
       data.key_field = key_field
       data.fields = fields[1..-1]
       data.type = :list
-      data.serializer = :list 
+      data.serializer ||= serializer
 
       database.with_unnamed do
         database.with_monitor(options[:monitor]) do

data/lib/rbbt/resource.rb

@@ -153,7 +153,9 @@ module Resource
             when :string
               Misc.sensiblewrite(final_path, content)
             when :url
-              Misc.sensiblewrite(final_path, Open.open(content))
+              options = {}
+              options[:noz] = true if Open.gzip?(final_path) || Open.bgzip?(final_path) || Open.zip?(final_path)
+              Misc.sensiblewrite(final_path, Open.open(content, options))
             when :proc
               data = case content.arity
                      when 0
@@ -192,7 +194,7 @@ source "$INSTALL_HELPER_FILE"
               EOF
 
               script = preamble + "\n" + Open.read(content)
-              install_io = CMD.cmd('bash', :in => script, :log => true, :pipe => true)
+              install_io = CMD.cmd('bash', :in => script, :log => true, :pipe => true, :stderr => true)
               while line = install_io.gets
                 Log.debug line
               end

data/lib/rbbt/tsv.rb

@@ -128,10 +128,12 @@ module TSV
     case
     when Path === stream 
       stream.open do |f|
+        f.no_fail = false if ConcurrentStream === f
         Parser.new f, options
       end
     when (String === stream and stream.length < 300 and (Open.exists? stream or Open.remote? stream))
       Open.open(stream) do |f|
+        f.no_fail = false if ConcurrentStream === f
         Parser.new f, options
       end
     else

data/lib/rbbt/tsv/accessor.rb

@@ -193,14 +193,14 @@ module TSV
   def serializer=(serializer)
     @serializer = serializer
     self.send(:[]=, KEY_PREFIX + 'serializer', dump_entry_value(serializer), :entry_key)
-    @serializar_module = serializer.nil? ? TSV::CleanSerializer : SERIALIZER_ALIAS[serializer.to_sym]
+    @serializar_module = serializer.nil? ? TSV::CleanSerializer : (Module === serializer ? serializer : SERIALIZER_ALIAS[serializer.to_sym])
   end
 
 
   def serializer_module
     @serializer_module ||= begin
                              serializer = self.serializer
-                             mod = serializer.nil? ? TSV::CleanSerializer : SERIALIZER_ALIAS[serializer.to_sym]
+                             mod = serializer.nil? ? TSV::CleanSerializer : (Module === serializer ? serializer : SERIALIZER_ALIAS[serializer.to_sym])
                              raise "No serializer_module for: #{ serializer.inspect }" if mod.nil?
                              mod
                            end

data/lib/rbbt/tsv/parser.rb

@@ -222,6 +222,7 @@ module TSV
     end
 
     def add_to_data_no_merge_double(data, keys, values)
+      keys = [keys] unless Array === keys
       keys.each do |key|
         next if data.include? key
         data[key] = values 

data/lib/rbbt/tsv/serializers.rb

@@ -1,6 +1,5 @@
 module TSV
 
-
   class CleanSerializer
     def self.dump(o); o end
     def self.load(o); o end

data/lib/rbbt/util/cmd.rb

@@ -134,7 +134,7 @@ module CMD
     if pipe
       err_thread = Thread.new do
         while line = serr.gets
-          Log.log line, stderr if Integer === stderr and log
+          Log.log "STDERR [#{pid}]: " +  line, stderr if Integer === stderr and log
         end
         serr.close
       end
@@ -144,27 +144,47 @@ module CMD
       sout
     else
       err = ""
-      Thread.new do
+      err_thread = Thread.new do
         while not serr.eof?
           err << serr.gets if Integer === stderr
         end
-          serr.close
-        end
+        serr.close
+      end
 
-        ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => no_wait, :no_fail => no_fail 
+      ConcurrentStream.setup sout, :pids => pids, :threads => [in_thread, err_thread].compact, :autojoin => no_wait, :no_fail => no_fail 
 
-        out = StringIO.new sout.read
-        sout.close unless sout.closed?
+      out = StringIO.new sout.read
+      sout.close unless sout.closed?
 
-        Process.waitpid pid
+      Process.waitpid pid
 
-        if not $?.success? and not no_fail
-          raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{$?.exitstatus}.\n#{err}"
-        else
-          Log.log err, stderr if Integer === stderr and log
-        end
+      if not $?.success? and not no_fail
+        raise ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{$?.exitstatus}.\n#{err}"
+      else
+        Log.log err, stderr if Integer === stderr and log
+      end
+
+      out
+    end
+  end
 
-        out
+  def self.cmd_log(*args)
+    all_args = *args
+
+    all_args << {} unless Hash === all_args.last
+    all_args.last[:log] = true
+    all_args.last[:pipe] = true
+
+    io = cmd(*all_args)
+    pid = io.pids.first
+    while line = io.gets
+      if pid
+        Log.debug "STDOUT [#{pid}]: " + line
+      else
+        Log.debug "STDOUT: " + line
+      end
     end
+    io.join
+    nil
   end
 end

data/lib/rbbt/util/concurrency/processes.rb

@@ -82,6 +82,12 @@ class RbbtProcessQueue
       rescue Exception
         Log.warn "Process monitor exception: #{$!.message}"
         @processes.each{|p| p.abort }
+        @processes.each{|p| 
+          begin
+            p.join 
+          rescue ProcessFailed
+          end
+        }
         @callback_thread.raise $! if @callback_thread and @callback_thread.alive?
         raise $!
       end

data/lib/rbbt/util/concurrency/processes/worker.rb

@@ -114,7 +114,9 @@ class RbbtProcessQueue
           Log.high "Worker #{Process.pid} respawning to #{@current}"
         end
       rescue Aborted, Interrupt
-        Log.warn "Worker #{Process.pid} aborted"
+        Log.warn "Worker #{Process.pid} aborted. Current #{@current} #{Misc.pid_exists?(@current) ? "exists" : "does not exist"}"
+        Process.kill "INT", @current if Misc.pid_exists? @current
+        @callback_queue.close_write if @callback_queue 
         Kernel.exit! 0
       rescue Exception
         Log.exception $!

data/lib/rbbt/util/misc/concurrent_stream.rb

@@ -81,8 +81,12 @@ module ConcurrentStream
         begin
           t.join #unless FalseClass === t.status 
         rescue Exception
-          Log.warn "Exception joining thread in ConcurrenStream: #{filename}"
-          raise $!
+          if no_fail
+            Log.low "Not failing on exception joining thread in ConcurrenStream: #{filename}"
+          else
+            Log.warn "Exception joining thread in ConcurrenStream: #{filename}"
+            raise $! 
+          end
         end
       end
     end
@@ -190,7 +194,7 @@ module ConcurrentStream
     end
   end
 
-  def super(*args)
+  def close(*args)
     if autojoin
       begin
         super(*args)

data/lib/rbbt/util/misc/development.rb

@@ -320,12 +320,12 @@ def self.add_libdir(dir=nil)
     TSV.traverse index, options do |pos|
       elem = elems[pos.to_i]
       elems.annotate elem if elems.respond_to? :annotate
-      begin
-        res = yield elem
-      rescue Interrupt
-        Log.warn "Process #{Process.pid} was aborted"
-        raise $!
-      end
+      res = begin
+              yield elem
+            rescue Interrupt
+              Log.warn "Process #{Process.pid} was aborted"
+              raise $!
+            end
       res = nil unless options[:into]
       raise RbbtProcessQueue::RbbtProcessQueueWorker::Respawn, res if respawn == :always and cpus > 1
       res

data/lib/rbbt/util/misc/exceptions.rb

@@ -23,6 +23,8 @@ class DependencyError < Aborted
   end
 end
 
+class DontClose < Exception; end
+
 class KeepLocked < Exception
   attr_accessor :payload
   def initialize(payload)

data/lib/rbbt/util/misc/objects.rb

@@ -63,16 +63,36 @@ module Misc
     current
   end
 
-  def self.zip_fields(array)
+  def self._zip_fields(array, max = nil)
     return [] if array.nil? or array.empty? or (first = array.first).nil?
-    max = array.collect{|l| l.length}.max
+    max = array.collect{|l| l.length}.max if max.nil?
     rest = array[1..-1].collect{|v|
       v.length == 1 & max > 1 ? v * max : v
     }
     first = first * max if first.length == 1 and max > 1
+
     first.zip(*rest)
   end
 
+  def self.zip_fields(array)
+    if array.length < 10000
+      _zip_fields(array)
+    else
+      zipped_slices = []
+      max = array.collect{|l| l.length}.max
+      array.each_slice(10000) do |slice|
+        zipped_slices << _zip_fields(slice, max)
+      end
+      new = zipped_slices.first
+      zipped_slices[1..-1].each do |rest|
+        rest.each_with_index do |list,i|
+          new[i].concat list
+        end
+      end
+      new
+    end
+  end
+
   def self.choose(array, select)
     array.zip(select).select{|e,s| s }.collect{|e,s| e }
   end

data/lib/rbbt/util/misc/pipes.rb

@@ -32,6 +32,17 @@ module Misc
     Log.debug{"Creating pipe #{[res.last.inspect,res.first.inspect] * " => "}"}
     res
   end
+
+  def self.with_fifo(path = nil, &block)
+    begin
+      erase = path.nil?
+      path = TmpFile.tmp_file if path.nil?
+      File.mkfifo path
+      yield path
+    ensure
+      FileUtils.rm path if erase
+    end
+  end
   
   def self.release_pipes(*pipes)
     PIPE_MUTEX.synchronize do
@@ -73,6 +84,7 @@ module Misc
         end
         Kernel.exit! 0
       }
+      sin.close
 
       ConcurrentStream.setup sout, :pids => [pid]
     else
@@ -207,6 +219,8 @@ module Misc
       ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename, :_pair => stream
     end
 
+    out_pipes.first.autojoin = true
+
     out_pipes.first.callback = Proc.new do 
       stream.join
       in_pipes[1..-1].each do |sin|

data/lib/rbbt/util/misc/system.rb

@@ -78,7 +78,7 @@ end
     old_pwd = FileUtils.pwd
     res = nil
     begin
-      FileUtils.mkdir_p dir unless File.exist? dir
+      FileUtils.mkdir_p dir unless File.exist?(dir)
       FileUtils.cd dir
       res = yield
     ensure

data/lib/rbbt/util/open.rb

@@ -393,27 +393,26 @@ module Open
     io = gunzip(io) if ((String === url and gzip?(url)) and not options[:noz]) or options[:gzip]
     io = bgunzip(io) if ((String === url and bgzip?(url)) and not options[:noz]) or options[:bgzip]
 
+    class << io;
+      attr_accessor :filename
+    end
+
+    io.filename = url.to_s
+
     if block_given?
       begin
         return yield(io)
+      rescue DontClose
       rescue Exception
         io.abort if io.respond_to? :abort
         io.join if io.respond_to? :join
         raise $!
       ensure
-        io.join if io.respond_to? :join
         io.close if io.respond_to? :close and not io.closed?
+        io.join if io.respond_to? :join
       end
-    else
-      io
     end
 
-    class << io;
-      attr_accessor :filename
-    end
-
-    io.filename = url.to_s
-
     io
   end
 

data/lib/rbbt/workflow/accessor.rb

@@ -770,7 +770,7 @@ module Workflow
         rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
 
         if rec_dependency.nil?
-          if inputs.include? v
+          if _inputs.include? v
             _inputs[i] = _inputs.delete(v)
           else
             _inputs[i] = v unless _inputs.include? i

data/share/install/software/lib/install_helpers

@@ -140,6 +140,10 @@ build(){
     echo `pwd`
     ls
 
+    if [ -f Makefile.am -a -n -f configure ]; then
+      autoreconf
+    fi
+
     if [ -f aclocal.m4 ]; then
         autoconf
     fi

data/share/rbbt_commands/workflow/cmd

@@ -75,9 +75,7 @@ begin
         exit 0
       end
 
-      puts "Error: Command not understood"
-      puts
-      break
+      raise ParameterException, "Error: Command not understood: #{command}"
     end
   end
 end

data/share/rbbt_commands/workflow/server

@@ -1,3 +1,4 @@
+#!/usr/bin/env ruby
 
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
@@ -22,6 +23,7 @@ $ rbbt workflow server [options] <Workflow>
 -R--requires* Require a list of files
 --views* Directory with view templates
 --stream Activate streaming of workflow tasks
+--export_all Export all workflow tasks (use with caution!)
 --export* Export workflow tasks (asynchronous)
 --export_asynchronous* Export workflow tasks as asynchronous
 --export_synchronous* Export workflow tasks as synchronous
@@ -83,6 +85,10 @@ TmpFile.with_file do |app_dir|
     ENV["RACK_ENV"] = options[:environment] if options.include?(:environment)
     ENV["RBBT_VIEWS_DIR"] = options[:views] if options.include?(:views)
 
+    if options[:export_all]
+      ENV["RBBT_WORKFLOW_EXPORT_ALL"] = 'true'
+    end
+
     if options[:stream]
       raise "No streaming available for any server other than puma" unless true or options[:server].include? 'puma'
       ENV["RBBT_WORKFLOW_TASK_STREAM"] = 'true'

data/test/rbbt/util/test_misc.rb

@@ -171,6 +171,7 @@ eum fugiat quo voluptas nulla pariatur?"
         sleep 0.5
         sin.puts "LINE #{ i }"
       end
+      sin.close
     end
 
     time = Time.now
@@ -240,12 +241,14 @@ eum fugiat quo voluptas nulla pariatur?"
         sin.puts line.strip.reverse
       end
     end
+    stream2.close
 
     stream5 = Misc.open_pipe(true) do |sin|
       while line = stream3.gets
         sin.puts line.strip.downcase
       end
     end
+    stream3.close
 
     lines1 = []
     th1 = Thread.new do
@@ -369,6 +372,13 @@ eum fugiat quo voluptas nulla pariatur?"
     assert_equal current, Misc.zip_fields(Misc.zip_fields(current))
   end
 
+  def test_zip_fields_large
+    current = (1..1500).to_a.collect{|i| ["A#{i}", "B#{i}"]}
+    assert_equal current, Misc.zip_fields(Misc.zip_fields(current))
+    current = (1..150000).to_a.collect{|i| ["A#{i}", "B#{i}"]}
+    assert_equal current, Misc.zip_fields(Misc.zip_fields(current))
+  end
+
   def test_zip_fields_comp
     current = [[:a,1], [:b,2], [:c]]
     assert_equal [[:a, :b, :c],[1,2,nil]], Misc.zip_fields(current)
@@ -423,7 +433,8 @@ eum fugiat quo voluptas nulla pariatur?"
   end
 
   def test_tarize
-    path = File.expand_path('/home/mvazquezg/git/rbbt-util/lib')
+    dir = Path.caller_lib_dir(__FILE__)
+    path = File.expand_path(dir)
     stream = Misc.tarize(path)
     TmpFile.with_file do |res|
       Misc.in_dir(res) do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.129
+  version: 5.21.130
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-14 00:00:00.000000000 Z
+date: 2018-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake