rbbt-util 5.27.13 → 5.28.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99b1ad9e8d9403c7f77e22b16497456324c340b08122b765033530f713e35c3c
-  data.tar.gz: 2d33590e395b9ca715e1c16d509d0a62a1ebeab2315b700ff4377a08803c89af
+  metadata.gz: 2dc2c06d6d3e5f783f24a671c0150d6cb2da641d003770ee5315c10118a0ea34
+  data.tar.gz: 0e5cf607bd69452be51bf21d36ceeef49ab3d0cb2821728900547e7db60a03af
 SHA512:
-  metadata.gz: '097ce6c8c16c17441f5a8748ba4ab33afecf6d22d4d54c7ce53475cd642407ae5013346ba482f7fa9375176ab6790febfe030e9f7959ba7b928b5d3762382237'
-  data.tar.gz: 2fb616d7d66fc2eb979c9911921faecf163f51dd3323ac127265bcaefd0e8a4e3aab9ef41cc9bdf8bd351d44d4ef2cc388c692d5dc9a2d1b811155f21e3f0889
+  metadata.gz: 6d949e77540e01389eb90a757dbf7a499e6aadf237b998b533ffb964fee46bf2ff9c7145c00fa355ce4ff30a1710caab9169bebbf5be22a998c74cc84610c11e
+  data.tar.gz: c2d18ea2d6b45db2878c4a8ae07ac433d1ed365813bf7fe5635c325b743415575998ed6ae3db64d6429719c369ab8be5f263149d9fee8d3ca6edba15b0da1265

data/lib/rbbt/annotations/util.rb

@@ -74,7 +74,7 @@ module Annotated
 
     object = case
              when literal_pos
-               values[literal_pos]
+               values[literal_pos].tap{|o| o.force_encoding(Encoding.default_external)}
              else
                id.dup
              end
@@ -119,7 +119,7 @@ module Annotated
                fields = AnnotatedArray === annotations ? annotations.annotations : annotations.compact.first.annotations
                fields << :annotation_types
 
-             when (fields == [:literal] and not annotations.compact.empty?)
+             when (fields == [:literal] and ! annotations.compact.empty?)
                fields << :literal
 
              when (fields == [:all] && Annotated === annotations)

data/lib/rbbt/hpc.rb

@@ -451,12 +451,15 @@ EOF
           out = CMD.cmd("tail -f '#{fout}'", :pipe => true) if File.exists?(fout) and not tail == :STDERR
           err = CMD.cmd("tail -f '#{ferr}'", :pipe => true) if File.exists?(ferr)
 
-          Misc.consume_stream(err, true, STDERR) if err
-          Misc.consume_stream(out, true, STDOUT) if out
+          terr = Misc.consume_stream(err, true, STDERR) if err
+          tout = Misc.consume_stream(out, true, STDOUT) if out
 
           sleep 3 while CMD.cmd("squeue --job #{job}").read.include? job.to_s
+        rescue Aborted
         ensure
           begin
+            terr.exit if terr
+            tout.exit if tout
             err.close if err
             err.join if err
           rescue Exception

data/lib/rbbt/resource/path.rb

@@ -13,7 +13,7 @@ module Path
     string.resource = resource
     string.search_paths = search_paths
     string.search_order = search_order
-    string.libdir = libdir || Path.caller_lib_dir
+    string.libdir = libdir || Path.caller_lib_dir 
     string
   end
 
@@ -52,11 +52,11 @@ module Path
   def join(name)
     raise "Invalid path: #{ self }" if self.nil?
     new = if self.empty?
-            self.annotate name.to_s.dup
+            self.annotate name.to_s.dup.chomp
           else
-            self.annotate File.join(self, name.to_s)
+            self.annotate File.join(self, name.to_s.chomp)
           end
-    new.original = File.join(self.original, name.to_s) if self.original
+    new.original = File.join(self.original, name.to_s.chomp) if self.original
     new
   end
 
@@ -76,7 +76,7 @@ module Path
       return [] unless self.exists? 
       found = self.find
       exp = File.join(found, pattern)
-      paths = Dir.glob(exp).collect{|f| Path.setup(f, self.resource, self.pkgdir)}
+      paths = Dir.glob(exp).collect{|f| self.annotate(f) }
 
       paths.each do |p|
         p.original = File.join(found.original, p.sub(/^#{found}/, ''))
@@ -412,7 +412,9 @@ module Path
   end
 
   def replace_extension(new_extension = nil, multiple = false)
-    if multiple
+    if String === multiple
+      new_path = self.sub(/(\.[^\.\/]{1,5})(.#{multiple})?$/,'')
+    elsif multiple
       new_path = self.sub(/(\.[^\.\/]{1,5})+$/,'')
     else
       new_path = self.sub(/\.[^\.\/]{1,5}$/,'')

data/lib/rbbt/resource/util.rb

@@ -1,6 +1,6 @@
 module Path
 
-  def self.caller_lib_dir(file = nil, relative_to = 'lib')
+  def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
     file = caller.reject{|l| 
       l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
       l =~ /rbbt\/resource\/path\.rb/ or
@@ -9,13 +9,14 @@ module Path
       l =~ /progress-monitor\.rb/ 
     }.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
 
+    relative_to = [relative_to] unless Array === relative_to
     file = File.expand_path(file)
-    return Path.setup(file) if File.exist? File.join(file, relative_to)
+    return Path.setup(file) if relative_to.select{|d| File.exist? File.join(file, d)}.any?
 
     while file != '/'
       dir = File.dirname file
 
-      return dir if File.exist? File.join(dir, relative_to)
+      return dir if relative_to.select{|d| File.exist? File.join(dir, d)}.any?
 
       file = File.dirname file
     end

data/lib/rbbt/tsv/change_id.rb

@@ -58,13 +58,13 @@ module TSV
 
     identifiers, persist_input, compact = Misc.process_options options, :identifiers, :persist, :compact
     identifiers = tsv.identifier_files.first if identifiers.nil?
-    identifiers = Organism.identifiers(tsv.namespace) if identifiers.nil? and tsv.namespace and Organism.identifiers(tsv.namespace).exists?
+    identifiers = Organism.identifiers(tsv.namespace) if defined?(Organism) && identifiers.nil? && tsv.namespace && Organism.identifiers(tsv.namespace).exists?
     identifiers.namespace ||= tsv.namespace
 
     fields = (identifiers and identifiers.all_fields.include?(field))? [field] : nil 
     #index = identifiers.index :target => format, :fields => fields, :persist => persist_input, :order => true
 
-    grep = Organism.blacklist_genes(tsv.namespace).list  if identifiers.namespace and Organism.blacklist_genes(tsv.namespace).exists?
+    grep = Organism.blacklist_genes(tsv.namespace).list  if defined?(Organism) && identifiers.namespace && Organism.blacklist_genes(tsv.namespace).exists?
     if fields.nil?
       index = identifiers.index(:data_tsv_grep => grep, :data_invert_grep => true, :target => format, :persist => true, :order => true, :unnamed => true, :data_persist => true)
     else

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -101,8 +101,8 @@ module TSV
         error = true
         raise $!
       ensure
-        Log::ProgressBar.remove_bar(bar) if bar
         join.call(error) if join
+        Log::ProgressBar.remove_bar(bar) if bar
       end
   end
 
@@ -137,8 +137,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -177,8 +177,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -219,8 +219,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -273,8 +273,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -318,8 +318,8 @@ module TSV
       error = true
       raise $!
     ensure
-      Log::ProgressBar.remove_bar(bar) if bar
       join.call(error) if join
+      Log::ProgressBar.remove_bar(bar) if bar
     end
   end
 
@@ -727,8 +727,6 @@ module TSV
         Log.exception $!
         abort_stream(into, $!)
         raise $!
-      ensure
-        Log::ProgressBar.remove_bar(bar) if bar
       end
 
       into

data/lib/rbbt/tsv/util.rb

@@ -316,22 +316,30 @@ module TSV
 
   def to_single
     new = {}
-    case type
-    when :double
-      through do |k,v|
-        new[k] = v.first.first
-      end
-    when :flat
+
+    if block_given?
       through do |k,v|
-        new[k] = v.first
+        new[k] = yield v
       end
-    when :single
-      return self
-    when :list
-      through do |k,v|
-        new[k] = v.first
+    else
+      case type
+      when :double
+        through do |k,v|
+          new[k] = v.first.first
+        end
+      when :flat
+        through do |k,v|
+          new[k] = v.first
+        end
+      when :single
+        return self
+      when :list
+        through do |k,v|
+          new[k] = v.first
+        end
       end
     end
+
     self.annotate(new)
     new.type = :single
     new.fields = [new.fields.first] if new.fields.length > 1

data/lib/rbbt/util/misc/inspect.rb

@@ -290,7 +290,7 @@ module Misc
             if obj.exists?
               if obj.directory?
                 files = obj.glob("**/*")
-                "directory: #{files}"
+                "directory: #{Misc.fingerprint(files)}"
               else
                 "file: " << obj << "--" << mtime_str(obj)
               end
@@ -401,7 +401,7 @@ module Misc
 
   def self.scan_version_text(text, cmd = nil)
     cmd = "NOCMDGIVE" if cmd.nil? || cmd.empty?
-    m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
+    m = text.match(/(?:version.*?|#{cmd}.*?|#{cmd.to_s.split(/[-_.]/).first}.*?|v)((?:\d+\.)*\d+(?:-[a-z_]+)?)/i)
     return nil if m.nil?
     m[1]
   end

data/lib/rbbt/workflow/integration/cromwell.rb

@@ -1,13 +1,20 @@
-module Workflow
+module Cromwell
 
   Rbbt.claim Rbbt.software.opt.jar["cromwell.jar"], :url, "https://github.com/broadinstitute/cromwell/releases/download/48/cromwell-48.jar"
   Rbbt.claim Rbbt.software.opt.jar["wdltool.jar"], :url, "https://github.com/broadinstitute/wdltool/releases/download/0.14/wdltool-0.14.jar"
 
-  def run_cromwell(file, work_dir, options = {})
+  def self.run_cromwell(file, work_dir, options = {})
+    cromwell_inputs_file = Misc.process_options options, :cromwell_inputs_file
     jar = Rbbt.software.opt.jar["cromwell.jar"].produce.find
-    CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}'", options.merge("add_option_dashes" => true))
+    if cromwell_inputs_file
+      CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}' -i #{cromwell_inputs_file}", options.merge("add_option_dashes" => true))
+    else
+      CMD.cmd_log("java -jar '#{jar}' run '#{file}' --workflow-root='#{work_dir}'", options.merge("add_option_dashes" => true))
+    end
   end
+end
 
+module Workflow
   def load_cromwell(file)
     jar = Rbbt.software.opt.jar["wdltool.jar"].produce.find
     inputs = JSON.load(CMD.cmd("java -jar '#{jar}' inputs '#{file}'"))

data/lib/rbbt/workflow/remote_workflow/remote_step.rb

@@ -340,6 +340,11 @@ class RemoteStep < Step
     @inputs = new_inputs
     @info = nil
   end
+
+  def init_info(*args)
+    i = {:status => :waiting, :pid => Process.pid, :path => path}
+    i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
+  end
 end
 
 require 'rbbt/workflow/remote_workflow/remote_step/rest'

data/lib/rbbt/workflow/step/accessor.rb

@@ -208,7 +208,7 @@ class Step
   end
 
   def init_info(force = false)
-    return nil if @exec or info_file.nil? or (Open.exists?(info_file) and ! force)
+    return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
     Open.lock(info_file, :lock => info_lock) do
       i = {:status => :waiting, :pid => Process.pid, :path => path}
       i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
@@ -437,11 +437,12 @@ class Step
     rec_dependencies = self.rec_dependencies
     return [] if rec_dependencies.empty?
     canfail_paths = self.canfail_paths
+    dep = rec_dependencies.select{|d| d.task_name.to_s == 'contamination'}.first
     dirty_files = rec_dependencies.reject{|dep|
       (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dep) || 
         ! Open.exists?(dep.info_file) ||
         (dep.path && (Open.exists?(dep.path) || Open.remote?(dep.path))) || 
-        ((dep.error? || dep.aborted? || dep.waiting?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
+        ((dep.error? || dep.aborted?) && (! dep.recoverable_error? || canfail_paths.include?(dep.path)))
     }
   end
 
@@ -508,12 +509,12 @@ class Step
 
   def nopid?
     pid = info[:pid] || Open.exists?(pid_file)
-    ! pid && ! (status.nil? || status == :aborted || status == :done || status == :error)
+    ! pid && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
   end
 
   def aborted?
     status = self.status
-    status == :aborted || ((status != :noinfo && status != :setup && status != :noinfo) && nopid?)
+    status == :aborted || ((status != :cleaned && status != :noinfo && status != :setup && status != :noinfo) && nopid?)
   end
 
   # {{{ INFO

data/lib/rbbt/workflow/step/dependencies.rb

@@ -24,7 +24,7 @@ class Step
           Log.medium "Not duplicating stream #{stream_key}"
           STREAM_CACHE[stream_key] = stream
         when File
-          if Open.exists? current.path 
+          if Open.exists?(current.path)
             Log.medium "Reopening file #{stream_key}"
             Open.open(current.path)
           else
@@ -97,7 +97,7 @@ class Step
       end
 
       job.dup_inputs unless status == 'done' or job.started?
-      job.init_info unless status == 'noinfo' or status == 'done' or job.started?
+      job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started?
 
       canfail = ComputeDependency === job && job.canfail?
     end

data/share/rbbt_commands/app/start

@@ -54,9 +54,9 @@ app = ARGV.shift
 
 ENV["RServe-session"] = options[:RServe_session] || app
 
-app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : Rbbt.apps.find
+app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : Rbbt.apps
 
-app_dir = app_dir[app]
+app_dir = app_dir[app].find
 
 server = options[:server] || 'puma'
 Misc.in_dir(app_dir) do

data/share/workflow_config.ru

@@ -43,7 +43,11 @@ etc_dir = Rbbt.etc
 load_file etc_dir['app.d/pre.rb'].find 
 
 app.get '/' do
-  redirect to(File.join('/', wf.to_s))
+  begin
+    template_render('main', params, 'main', :cache_type => :asynchronous)
+  rescue TemplateMissing
+    redirect to(File.join('/', wf.to_s))
+  end
 end
 
 #{{{ BASE

data/test/rbbt/test_resource.rb

@@ -69,11 +69,11 @@ class TestTSV < Test::Unit::TestCase
   end
 
   def test_libdir
-    assert File.exist? TestResource[].share.Rlib["util.R"].find :lib
-    assert File.exist? TestResource[].share.Rlib["util.R"].find 
+    assert File.exist?(TestResource[].share.Rlib["util.R"].find(:lib))
+    assert File.exist?(TestResource[].share.Rlib["util.R"].find)
   end
 
-  def test_server
+  def __test_server
     require 'rbbt/sources/organism'
     TmpFile.with_file :extension => 'gz' do |tmp|
       Organism.get_from_server("Hsa/b37/known_sites/dbsnp_138.vcf.gz", tmp, 'http://rbbt.bsc.es')

data/test/rbbt/tsv/parallel/test_traverse.rb

@@ -202,7 +202,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     stream = datafile_test('identifiers').open 
     dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
     dumper.init
-    TSV.traverse stream, :head => head, :into => dumper do |k,v|
+    TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
       k = k.first
       [k,v]
     end
@@ -212,6 +212,26 @@ class TestTSVParallelThrough < Test::Unit::TestCase
     assert_equal head, res.size
   end
 
+  def test_traverse_into_dumper_error_bar
+    require 'rbbt/sources/organism'
+
+    head = 2_000
+
+    stream = datafile_test('identifiers').open 
+    dumper = TSV::Dumper.new datafile_test('identifiers').tsv_options
+    dumper.init
+    TSV.traverse stream, :head => head, :into => dumper, :bar => true do |k,v|
+      k = k.first
+      raise 
+      [k,v]
+    end
+
+    assert_raise do 
+      res = TSV.open(dumper.stream)
+    end
+
+  end
+
   def test_traverse_into_dumper_threads
     require 'rbbt/sources/organism'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.27.13
+  version: 5.28.5
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-16 00:00:00.000000000 Z
+date: 2020-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake