rbbt-util 5.27.11 → 5.28.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be6c76069ffd5a6a49876a42c95f42e7c63975e287446d786c7e4db154456790
-  data.tar.gz: c6bdbb33d38b424dc6355e6dc70a15a4bfefc65b7331d2eb471476b8522ac3a3
+  metadata.gz: 6bc94b86c69f95806ae280d76df8da7696f5df0f008a3791385e4ccf11b70977
+  data.tar.gz: f7763e2dd88ba121b8a625b846bf0f4f1efaba090b873f2947e0a83ae9dabff1
 SHA512:
-  metadata.gz: 1bccd1104a3b2e6236073ca6cd267dba65c0b51a16a6af87c0ea09e1361b8eaf7a9b1a6c7d9bb7e35da3ab4a235ebb578ccd9ce64fd1cdf562282d81a78a3edf
-  data.tar.gz: edf6d4339a6ddafbc0e88d8356686fe7986ae5983ff4a7905be45f419b8d4887b0ed44332f4a537da6cf3e8a3909ce6c687dd0745c70485ff00a7231f9ee0062
+  metadata.gz: 59fd76fc00eeeb892ad53bbbc0757d5e17dde600f505d31e8dabe43e0184e5c5bfcef2ccc3f7aa9bc295e36251b9f864706bc0dcf357cc3606b73b1f25f7d672
+  data.tar.gz: ce4d345fc9876378b57ba8753d2a7dfd13b4fd23330246bc2385186f81344026cf415e7cc6eb86253f86e709e2061139930734b2c77b7329165ed2892e3e5d0a

data/lib/rbbt/annotations/util.rb

@@ -74,7 +74,7 @@ module Annotated
 
     object = case
              when literal_pos
-               values[literal_pos]
+               values[literal_pos].tap{|o| o.force_encoding(Encoding.default_external)}
              else
                id.dup
              end
@@ -119,7 +119,7 @@ module Annotated
                fields = AnnotatedArray === annotations ? annotations.annotations : annotations.compact.first.annotations
                fields << :annotation_types
 
-             when (fields == [:literal] and not annotations.compact.empty?)
+             when (fields == [:literal] and ! annotations.compact.empty?)
                fields << :literal
 
              when (fields == [:all] && Annotated === annotations)

data/lib/rbbt/resource/path.rb

@@ -13,7 +13,7 @@ module Path
     string.resource = resource
     string.search_paths = search_paths
     string.search_order = search_order
-    string.libdir = libdir || Path.caller_lib_dir
+    string.libdir = libdir || Path.caller_lib_dir 
     string
   end
 
@@ -52,11 +52,11 @@ module Path
   def join(name)
     raise "Invalid path: #{ self }" if self.nil?
     new = if self.empty?
-            self.annotate name.to_s.dup
+            self.annotate name.to_s.dup.chomp
           else
-            self.annotate File.join(self, name.to_s)
+            self.annotate File.join(self, name.to_s.chomp)
           end
-    new.original = File.join(self.original, name.to_s) if self.original
+    new.original = File.join(self.original, name.to_s.chomp) if self.original
     new
   end
 
@@ -76,7 +76,7 @@ module Path
       return [] unless self.exists? 
       found = self.find
       exp = File.join(found, pattern)
-      paths = Dir.glob(exp).collect{|f| Path.setup(f, self.resource, self.pkgdir)}
+      paths = Dir.glob(exp).collect{|f| self.annotate(f) }
 
       paths.each do |p|
         p.original = File.join(found.original, p.sub(/^#{found}/, ''))

data/lib/rbbt/resource/util.rb

@@ -1,6 +1,6 @@
 module Path
 
-  def self.caller_lib_dir(file = nil, relative_to = 'lib')
+  def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
     file = caller.reject{|l| 
       l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
       l =~ /rbbt\/resource\/path\.rb/ or
@@ -9,13 +9,14 @@ module Path
       l =~ /progress-monitor\.rb/ 
     }.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
 
+    relative_to = [relative_to] unless Array === relative_to
     file = File.expand_path(file)
-    return Path.setup(file) if File.exist? File.join(file, relative_to)
+    return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
 
     while file != '/'
       dir = File.dirname file
 
-      return dir if File.exist? File.join(dir, relative_to)
+      return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
 
       file = File.dirname file
     end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -101,8 +101,8 @@ module TSV
         error = true
         raise $!
       ensure
-        Log::ProgressBar.remove_bar(bar) if bar
         join.call(error) if join
+        Log::ProgressBar.remove_bar(bar) if bar
       end
   end
 
@@ -137,8 +137,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -177,8 +177,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -219,8 +219,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -273,8 +273,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -318,8 +318,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 

data/lib/rbbt/util/misc.rb

@@ -2,6 +2,7 @@ require 'lockfile'
 require 'digest/md5'
 require 'cgi'
 require 'zlib'
+require 'etc'
 require 'rubygems/package'
 
 require 'rbbt/util/tar'

data/lib/rbbt/util/misc/development.rb

@@ -295,13 +295,18 @@ def self.add_libdir(dir=nil)
   def self.bootstrap(elems, num = nil, options = {}, &block)
     IndiferentHash.setup options
 
-    num = Rbbt::Config.get :cpus, :default_bootstrap_cpus, :bootstrap_cpus if num == :current || num == nil
+    num = Rbbt::Config.get :cpus, :default_bootstrap_cpus, :bootstrap_cpus if num == :current || num == nil if defined?(Rbbt::Config)
     num = :current if num.nil?
     cpus = case num
            when :current
              n = Etc.nprocessors
              n = elems.length / 2 if n > elems.length/2
-             n
+
+             if $BOOTSTRAPPED_CURRENT && $BOOTSTRAPPED_CURRENT + n > Etc.nprocessors
+               1
+             else
+               n
+             end
            when String
              num.to_i
            when Integer
@@ -320,7 +325,12 @@ def self.add_libdir(dir=nil)
     respawn = options[:respawn] and options[:cpus] and options[:cpus].to_i > 1
 
     index = (0..elems.length-1).to_a.collect{|v| v.to_s }
+
     TSV.traverse index, options do |pos|
+      if num == :current
+        $BOOTSTRAPPED_CURRENT ||= n 
+        $BOOTSTRAPPED_CURRENT += 0 
+      end
       elem = elems[pos.to_i]
       elems.annotate elem if elems.respond_to? :annotate
       res = begin

data/lib/rbbt/util/misc/inspect.rb

@@ -290,7 +290,7 @@ module Misc
             if obj.exists?
               if obj.directory?
                 files = obj.glob("**/*")
-                "directory: #{files}"
+                "directory: #{Misc.fingerprint(files)}"
               else
                 "file: " << obj << "--" << mtime_str(obj)
               end
@@ -401,7 +401,7 @@ module Misc
 
   def self.scan_version_text(text, cmd = nil)
     cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
-    m = text.match(/(?:version.*?|#{cmd}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+    m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
     return nil if m.nil?
     m[1]
   end

data/lib/rbbt/util/open.rb

@@ -94,7 +94,8 @@ module Open
              end
 
     begin
-      wget_options = options.merge( '-O' => '-')
+      wget_options = options.dup
+      wget_options = wget_options.merge( '-O' => '-') unless options.include?('--output-document')
       wget_options[:pipe] = pipe unless pipe.nil?
       wget_options[:stderr] = stderr unless stderr.nil?
 
@@ -824,4 +825,8 @@ module Open
   def self.broken_link?(path)
     File.symlink?(path) && ! File.exists?(File.readlink(path))
   end
+
+  def self.download(url, path)
+    Open.wget(url, "--output-document" => path)
+  end
 end

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -340,6 +340,11 @@ class RemoteStep < Step
     @inputs = new_inputs
     @info = nil
   end
+
+  def init_info(*args)
+    i = {:status => :waiting, :pid => Process.pid, :path => path}
+    i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
+  end
 end
 
 require 'rbbt/workflow/remote_workflow/remote_step/rest'

data/lib/rbbt/workflow/step/accessor.rb

@@ -208,7 +208,7 @@ class Step
   end
 
   def init_info(force = false)
-    return nil if @exec or info_file.nil? or (Open.exists?(info_file) and ! force)
+    return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
       i = {:status => :waiting, :pid => Process.pid, :path => path}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies

data/lib/rbbt/workflow/step/dependencies.rb

@@ -24,7 +24,7 @@ class Step
           Log.medium "Not duplicating stream #{stream_key}"
           STREAM_CACHE[stream_key] = stream
         when File
-          if Open.exists? current.path 
+          if Open.exists?(current.path)
             Log.medium "Reopening file #{stream_key}"
             Open.open(current.path)
           else
@@ -97,7 +97,7 @@ class Step
       end
 
       job.dup_inputs unless status == 'done' or job.started?
-      job.init_info unless status == 'noinfo' or status == 'done' or job.started?
+      job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started?
 
       canfail = ComputeDependency === job && job.canfail?
     end

data/share/rbbt_commands/app/start

@@ -15,6 +15,7 @@ $ rbbt app start [options] <app_name>
 -Ho--Host* Host name
 -B--Bind* Bind IP
 -p--port* TCP port
+-w--workers* Number of workers for cluster mode (puma)
 -s--server* Server type:  thin, webrick, unicorn, etc
 -f--finder Start server with finder functionality
 -fs--file_server Activate file serving for resources
@@ -88,6 +89,8 @@ Misc.in_dir(app_dir) do
   options.each do |k,v| fixed_options[k.to_sym] = v end
   options = fixed_options
 
+  options[:workers] = options[:workers].to_i if options[:workers]
+
   case server
   when 'passenger'
     system ENV, "env RBBT_LOG=0 passenger start -R '#{config_ru_file}' -p #{options[:Port] || "2887"}"
@@ -95,6 +98,7 @@ Misc.in_dir(app_dir) do
     system ENV, "puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload"
   else
     options[:config] = config_ru_file
+    options[:threads] = "8:8"
     Rack::Server.start(options)
   end
 end

data/share/rbbt_commands/tsv/uncollapse

@@ -25,5 +25,4 @@ file = STDIN if file == '-'
 
 tsv = TSV.open file, :merge => true, :type => :double
 
-
 puts tsv.to_unmerged_expanded_s

data/share/workflow_config.ru

@@ -43,7 +43,11 @@ etc_dir = Rbbt.etc
 load_file etc_dir['app.d/pre.rb'].find 
 
 app.get '/' do
-  redirect to(File.join('/', wf.to_s))
+  begin
+    template_render('main', params, 'main', :cache_type => :asynchronous)
+  rescue TemplateMissing
+    redirect to(File.join('/', wf.to_s))
+  end
 end
 
 #{{{ BASE

data/test/rbbt/test_resource.rb

@@ -69,11 +69,11 @@ class TestTSV < Test::Unit::TestCase
   end
 
   def test_libdir
-    assert File.exist? TestResource[].share.Rlib["util.R"].find :lib
-    assert File.exist? TestResource[].share.Rlib["util.R"].find 
+    assert File.exist?(TestResource[].share.Rlib["util.R"].find(:lib))
+    assert File.exist?(TestResource[].share.Rlib["util.R"].find)
   end
 
-  def test_server
+  def __test_server
     require 'rbbt/sources/organism'
     TmpFile.with_file :extension => 'gz' do |tmp|
       Organism.get_from_server("Hsa/b37/known_sites/dbsnp_138.vcf.gz", tmp, 'http://rbbt.bsc.es')

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -202,7 +202,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     stream = datafile_test('identifiers').open 
     dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
     dumper.init
-    TSV.traverse stream, :head => head, :into => dumper do |k,v|
+    TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
       k = k.first
       [k,v]
     end
@@ -212,6 +212,26 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
+  def test_traverse_into_dumper_error_bar
+    require 'rbbt/sources/organism'
+
+    head = 2_000
+
+    stream = datafile_test('identifiers').open 
+    dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
+    dumper.init
+    TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
+      k = k.first
+      raise 
+      [k,v]
+    end
+
+    assert_raise do 
+      res = TSV.open(dumper.stream)
+    end
+
+  end
+
   def test_traverse_into_dumper_threads
     require 'rbbt/sources/organism'
 

data/test/rbbt/util/test_misc.rb

@@ -690,5 +690,30 @@ Written by Mike Haertel and others, see <http://git.sv.gnu.org/cgit/grep.git/tre
     EOF
     assert_equal "3.1", Misc.scan_version_text(txt, "grep")
   end
+
+  def test_bootstrap_current_only_once
+
+     
+    max = Etc.nprocessors * 10
+    TmpFile.with_file do |f|
+      RbbtSemaphore.with_semaphore(1) do |sem|
+        Misc.bootstrap (1..max).to_a do
+          Open.open(f, :mode => 'a') do |sin|
+            sin.puts Process.pid.to_s
+          end
+          Misc.bootstrap (1..10).to_a do
+            RbbtSemaphore.synchronize(sem) do
+              Open.open(f, :mode => 'a') do |sin|
+                sin.puts [Process.ppid.to_s, Process.pid.to_s] * ":"
+              end
+            end
+          end
+        end
+      end
+      assert_equal max * 10 + max, Open.read(f).split("\n").length
+      assert_equal Etc.nprocessors * 2, Open.read(f).split("\n").uniq.length
+    end
+
+  end
 end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.27.11
+  version: 5.28.2
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-13 00:00:00.000000000 Z
+date: 2020-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake