rbbt-util 5.34.1 → 5.34.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: dce81f37830228a43c702909b856ebf661002d17b49614d67b6c04e16d339d4b
-  data.tar.gz: d7cb5ef82e50da2287c4bf9cf3912d66e36c27bb72a1c1c702d86cf7d1e8df78
+  metadata.gz: eda1db070a45b3361a0c5c6b02fd59e39b031ae4264771f015e4068a888d1f09
+  data.tar.gz: 0f9a4bfafce4180f1db402510f83a880e22f5a75d92bf7506781801ce61575a2
 SHA512:
-  metadata.gz: fd9ad6b40180ef8cdc5a8e8335f291e557543f14cdd0cbc6760e7dd6f1327a5a73b5be92b1c037ccd46bca8596c86e7bf6eb7221989e9edac52ad34e81ad4e45
-  data.tar.gz: 72798bb53863c93aa768edc6c523d34e682dc374ebf4fe0307590faddb9022237a49f0556de364e7139345b35504b6d19b9c134abf8d0ae751a38c1d2c0331f7
+  metadata.gz: 7738b49efed37124e80b5aa72b367e9d92d32dc13d64fceef5c452e5b50687882d5bc03106aa906c7f425937ceac72694f8eb8959e2c9d3402164b8618c4c639
+  data.tar.gz: de617b931b585915e029f8f23124b6ce5cccc9091dec25e2fc3fdd8c4d2eaf4e23bfce798c73b753cb08f590933c316ceb7deffafd8e8aca854ea175d5744014

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2013 Miguel Vázquez García
+Copyright (c) 2010-2022 Miguel Vázquez García
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/rbbt/association/database.rb

@@ -86,6 +86,8 @@ module Association
     info_fields = field_pos.collect{|f| f == :key ? :key : all_fields[f]}
     options = options.merge({:key_field => source_field, :fields =>  info_fields})
 
+    fields = field_headers if fields.nil?
+
     data = options[:data] || {}
     TmpFile.with_file do |tmpfile|
       tmp_data = Persist.open_database(tmpfile, true, :double, "HDB")

data/lib/rbbt/association/open.rb

@@ -24,7 +24,7 @@ module Association
       options = options.dup
       data.serializer = :double if data.respond_to? :serializer
 
-      tsv = Association.database(file, options.merge(:persist => true, :unnamed => true, :data => data, :type => :double))
+      tsv = Association.database(file, options.merge(:unnamed => true, :data => data, :type => :double))
 
       data
     end

data/lib/rbbt/resource.rb

@@ -346,9 +346,12 @@ url='#{url}'
     locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys)
     locations -= [:current, "current"]
     locations << :current
+    search_paths = IndiferentHash.setup(resource.search_paths)
     locations.uniq.each do |name|
-      pattern = resource.search_paths[name]
+      pattern = search_paths[name]
+      pattern = resource.search_paths[pattern] while Symbol === pattern
       next if pattern.nil?
+
       pattern = pattern.sub('{PWD}', Dir.pwd)
       if String ===  pattern and pattern.include?('{')
         regexp = "^" + pattern.gsub(/{([^}]+)}/,'(?<\1>[^/]+)') + "(?:/(?<REST>.*))?/?$"

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -696,7 +696,7 @@ module TSV
 
       options[:join] = Proc.new do |error|
         error = false if error.nil?
-        Log::ProgressBar.remove_bar(bar, error)
+        Log::ProgressBar.remove_bar(bar, error) if bar
       end if bar
 
       options[:callback] = Proc.new do |e|

data/lib/rbbt/tsv/parser.rb

@@ -650,7 +650,7 @@ module TSV
           end
         end
       ensure
-        Log::ProgressBar.remove_bar(progress_monitor)
+        Log::ProgressBar.remove_bar(progress_monitor) if progress_monitor
         stream.close unless stream.closed?
         stream.join if stream.respond_to? :join and not stream.joined?
       end

data/lib/rbbt/util/log/progress/report.rb

@@ -65,11 +65,17 @@ module Log
 
       thr = 0.0000001 if thr == 0
       
-      if mean.nil? or mean.to_i > 1
+      if mean.nil? or mean.to_i > 2
         str = "#{ Log.color :blue, thr.to_i.to_s } per sec."
         #str << " #{ Log.color :yellow, mean.to_i.to_s } avg. #{Log.color :yellow, @mean_max.to_i.to_s} max." if @mean_max > 0
       else
-        str = "#{ Log.color :blue, (1/thr).ceil.to_s } secs each"
+        if 1.0/thr < 1
+          str = "#{ Log.color :blue, (1.0/thr).round(2).to_s } secs each"
+        elsif 1.0/thr < 2
+          str = "#{ Log.color :blue, (1.0/thr).round(1).to_s } secs each"
+        else
+          str = "#{ Log.color :blue, (1/thr).ceil.to_s } secs each"
+        end
         #str << " #{ Log.color :yellow, (1/mean).ceil.to_s } avg. #{Log.color :yellow, (1/@mean_max).ceil.to_s} min." if @mean_max > 0
       end
 
@@ -184,6 +190,7 @@ module Log
       @last_time = Time.now
       @last_count = ticks
       @last_percent = percent if max and max > 0
+      Log::LAST.replace "progress"
       save if file
     end
 

data/lib/rbbt/util/misc/exceptions.rb

@@ -1,7 +1,8 @@
 class RbbtException < StandardError; end
 class ParameterException < RbbtException; end
-class FieldNotFoundError < RbbtException;end
-class ClosedStream < RbbtException; end
+
+class FieldNotFoundError < StandardError;end
+class ClosedStream < StandardError; end
 
 class ProcessFailed < StandardError; 
   def initialize(pid = Process.pid)
@@ -26,7 +27,7 @@ end
 class SemaphoreInterrupted < TryAgain; end
 class LockInterrupted < TryAgain; end
 
-class RemoteServerError < RbbtException; end
+class RemoteServerError < StandardError; end
 
 class DependencyError < Aborted
   def initialize(msg)

data/lib/rbbt/util/misc/inspect.rb

@@ -298,6 +298,8 @@ module Misc
     str = case obj
           when nil
             'nil'
+          when Numeric
+            obj.to_f
           when Symbol 
             obj.to_s
           when TrueClass

data/lib/rbbt/util/misc/pipes.rb

@@ -420,7 +420,7 @@ module Misc
           end
 
           Open.touch path if Open.exists? path
-          content.join if content.respond_to? :join and not (content.respond_to?(:joined?) and content.joined?)
+          content.join if content.respond_to?(:join) and not Path === content and not (content.respond_to?(:joined?) && content.joined?)
 
           Open.notify_write(path) 
         rescue Aborted

data/lib/rbbt/util/named_array.rb

@@ -22,7 +22,7 @@ module NamedArray
   def self.setup(array, fields, key = nil, entity_options = nil, entity_templates = nil)
     return array if array.nil?
     array.extend NamedArray unless NamedArray === array
-    array.fields = Annotated.purge fields
+    array.fields = Annotated === fields ? Annotated.purge(fields) : fields
     array.key = key
     array.entity_options = entity_options unless entity_options.nil?
     array.entity_templates = entity_templates unless entity_templates.nil?

data/lib/rbbt/util/python.rb

@@ -4,6 +4,35 @@ require 'pycall/import'
 module RbbtPython
   extend PyCall::Import
 
+  def self.script(text, options = {})
+    Log.debug "Running python script:\n#{text.dup}"
+    text = StringIO.new text unless IO === text
+    CMD.cmd_log(:python, options.merge(:in => text))
+  end
+
+  def self.add_path(path)
+    self.run 'sys' do
+      sys.path.append path
+    end
+  end
+
+  def self.add_paths(paths)
+    self.run 'sys' do
+      paths.each do |path|
+        sys.path.append path
+      end
+    end
+  end
+
+  def self.init_rbbt
+    if ! defined?(@@__init_rbbt) || ! @@__init_rbbt 
+      Log.debug "Loading python 'rbbt' module into pycall RbbtPython module"
+      RbbtPython.add_paths(Rbbt.python.find_all)
+      RbbtPython.pyimport("rbbt")
+      @@__init_rbbt = true
+    end
+  end
+
   def self.exec(script)
     PyCall.exec(script)
   end
@@ -63,7 +92,9 @@ module RbbtPython
 
   def self.run_log(mod = nil, imports = nil, severity = 0, severity_err = nil, &block)
     if mod
-      if Array === imports
+      if imports == "*" || imports == ["*"]
+        pyfrom mod
+      elsif Array === imports
         pyfrom mod, :import => imports
       elsif Hash === imports
         pyimport mod, imports
@@ -92,21 +123,4 @@ module RbbtPython
       module_eval(&block)
     end
   end
-
-  def self.add_path(path)
-    self.run 'sys' do
-      sys.path.append path
-    end
-  end
-
-  def self.add_paths(paths)
-    self.run 'sys' do
-      paths.each do |path|
-        sys.path.append path
-      end
-    end
-  end
-
-  RbbtPython.add_paths Rbbt.python.find_all
-  RbbtPython.pyimport "rbbt"
 end

data/lib/rbbt/workflow/definition.rb

@@ -22,8 +22,13 @@ module Workflow
     :extension          => nil)
 
 
-  def helper(name, &block)
-    helpers[name] = block
+  def helper(name, *args, &block)
+    if block_given?
+      helpers[name] = block
+    else
+      raise RbbtException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
+      helpers[name].call(*args)
+    end
   end
 
   def desc(description)

data/lib/rbbt/workflow/dependencies.rb

@@ -71,6 +71,7 @@ module Workflow
 
   def setup_override_dependency(dep, workflow, task_name)
     return [] if dep == :skip || dep == 'skip'
+
     dep = Step === dep ? dep.dup : Workflow.load_step(dep)
 
     dep.original_workflow ||= dep.workflow if dep.workflow

data/lib/rbbt/workflow/step/dependencies.rb

@@ -230,7 +230,14 @@ class Step
       if dep_step[step.path] and dep_step[step.path].length > 1
         stream = step.result
         other_steps = dep_step[step.path].uniq.reject{|d| d.overriden }
+
+        other_steps = other_steps.collect{|d|
+          deps_using_step_input = d.rec_dependencies.select{|d| d.inputs.include? step  }
+          deps_using_step_input.any? ? deps_using_step_input : d
+        }.flatten.uniq
+
         return unless other_steps.length > 1
+
         log_dependency_exec(step, "duplicating #{other_steps.length}") 
         copies = Misc.tee_stream_thread_multiple(stream, other_steps.length)
         copies.extend StreamArray
@@ -370,7 +377,6 @@ class Step
   end
 
   def run_dependencies
-    dep_step = {}
 
     rec_dependencies = self.rec_dependencies(true) + input_dependencies
 
@@ -385,23 +391,30 @@ class Step
 
     canfail_paths = self.canfail_paths
 
+    dep_step = {}
     seen_paths = Set.new
     all_deps.uniq.each do |step|
       next if seen_paths.include? step.path
       seen_paths << step.path
+
       begin
         Step.prepare_for_execution(step) unless step == self
       rescue DependencyError, DependencyRbbtException
         raise $! unless canfail_paths.include? step.path
       end
+
       next unless step.dependencies and step.dependencies.any?
-      (step.dependencies + step.input_dependencies).each do |step_dep|
+
+      # ToDo is this really necessary
+      #(step.dependencies + step.input_dependencies).each do |step_dep|
+      step.dependencies.each do |step_dep|
         next unless step.dependencies.include?(step_dep)
         next if step_dep.done? or step_dep.running? or 
           (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
         dep_step[step_dep.path] << step
       end
+
     end
 
     produced = []

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -125,86 +125,6 @@ module Workflow
     inputs = task_inputs_from_directory(task_name, directory)
     job(task_name, jobname, inputs)
   end
-
-  #def self.load_inputs_old(dir, input_names, input_types)
-  #  inputs = {}
-  #  if File.exists?(dir) && ! File.directory?(dir)
-  #    Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
-  #    tarfile = dir
-  #    digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
-  #    tmpdir = Rbbt.tmp.input_bundle[digest].find
-  #    Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
-  #    files = tmpdir.glob("*")
-  #    if files.length == 1 && File.directory?(files.first)
-  #      tmpdir = files.first
-  #    end
-  #    load_inputs(tmpdir, input_names, input_types)
-  #  else
-  #    dir = Path.setup(dir.dup)
-  #    input_names.each do |input|
-  #      file = dir[input].find
-  #      file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not (File.symlink?(file) || file.exists?)
-  #      Log.debug "Trying #{ input }: #{file}"
-  #      next unless file and (File.symlink?(file) || file.exists?)
-
-  #      type = input_types[input]
-
-  #      type = :io if file.split(".").last == 'as_io'
-
-  #      type = :path if file.split(".").last == 'as_path'
-
-  #      type = :filename if file.split(".").last == 'as_filename'
-
-  #      type = :nofile if file.split(".").last == 'nofile'
-
-  #      case type
-  #      when :nofile
-  #        inputs[input.to_sym]  = Open.realpath(file)
-  #      when :path
-  #        inputs[input.to_sym]  = Open.realpath(Open.read(file).strip)
-  #      when :io
-  #        inputs[input.to_sym] = Open.open(Open.realpath(file))
-  #      when :file, :binary
-  #        Log.debug "Pointing #{ input } to #{file}"
-  #        if file =~ /\.yaml/
-  #          inputs[input.to_sym]  = YAML.load(Open.read(file))
-  #        else
-  #          if File.symlink?(file)
-  #            link_target = File.expand_path(File.readlink(file), File.dirname(file))
-  #            inputs[input.to_sym]  = link_target
-  #          else
-  #            inputs[input.to_sym]  = Open.realpath(file)
-  #          end
-  #        end
-  #      when :text
-  #        Log.debug "Reading #{ input } from #{file}"
-  #        inputs[input.to_sym]  = Open.read(file)
-  #      when :array
-  #        Log.debug "Reading array #{ input } from #{file}"
-  #        inputs[input.to_sym]  = Open.read(file).split("\n")
-  #      when :tsv
-  #        Log.debug "Opening tsv #{ input } from #{file}"
-  #        inputs[input.to_sym]  = TSV.open(file)
-  #      when :boolean
-  #        inputs[input.to_sym]  = (file.read.strip == 'true')
-  #      else
-  #        Log.debug "Loading #{ input } from #{file}"
-  #        inputs[input.to_sym]  = file.read.strip
-  #      end
-
-  #    end
-  #    inputs = IndiferentHash.setup(inputs)
-
-  #    dir.glob("*#*").each do |od|
-  #      name = File.basename(od)
-  #      value = Open.read(od)
-  #      Log.debug "Loading override dependency #{ name } as #{value}"
-  #      inputs[name] = value.chomp
-  #    end
-
-  #    inputs
-  #  end
-  #end
 end
 
 class Step

data/share/Rlib/svg.R

@@ -20,9 +20,11 @@ rbbt.SVG.extract <- function(plot, size=NULL, prefix=NULL, ...){
         resolution = 72 * (size/7)
 
         if (length(plot$theme) == 0) plot <- plot + theme_light();
+
         if (length(plot$theme$text) == 0) plot <- plot + theme(text = element_text(size=base.size));
 
-        plot$theme$text$size = base.size
+        if (is.null(plot$theme$text$size))
+          plot$theme$text$size = base.size
 
         print(plot, type='cairo')
         mysvg <- grid.export(res=resolution, prefix=prefix, ...)

data/share/rbbt_commands/workflow/server

@@ -114,7 +114,7 @@ TmpFile.with_file do |app_dir|
         name, _sep, value = pair.partition("=")
         name = name[1..-1].to_sym if name[0] == ':'
         value = value.to_i if value =~ /^\d+$/
-        value = true if value == "true"
+        value = true if value.nil? || value == "true"
         value = false if value == "false"
         options[name] = value
       end

data/share/rbbt_commands/workflow/task

@@ -598,14 +598,14 @@ when Step
   elsif detach
     exit! 0
   else
-    res.join
+      res.join if res.running?
     if %w(float integer string boolean).include?(res.result_type.to_s)
       out.puts res.load
     else
       Open.open(res.path, :mode => 'rb') do |io|
         Misc.consume_stream(io, false, out)
       end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
-    end
+    end if res.done?
   end
 else
   if Array === res

data/test/rbbt/util/misc/test_omics.rb

@@ -88,8 +88,28 @@ class TestMiscOmics < Test::Unit::TestCase
       index = Misc.index_BED(io, dir)
       assert_equal ["2:2"], index["2:220:230"]
     end
+  end
 
+  def test_sort_genomic_locations
+    mutations =<<-EOF.split("\n").shuffle
+1:100:A
+1:20:A
+1:300:A
+2:100:A
+2:20:A
+2:300:A
+10:100:A
+10:20:A
+10:300:A
+    EOF
+    sorted =  Misc.sort_mutation_stream(StringIO.new(mutations * "\n")).read.split("\n")
+    strict_sorted = Misc.sort_mutation_stream_strict(StringIO.new(mutations * "\n")).read.split("\n")
 
-
+    assert sorted.index("1:20:A") < sorted.index("1:100:A")
+    assert sorted.index("1:300:A") < sorted.index("10:300:A")
+    assert sorted.index("10:300:A") < sorted.index("2:300:A")
+    assert strict_sorted.index("1:20:A") < strict_sorted.index("1:100:A")
+    assert strict_sorted.index("1:300:A") < strict_sorted.index("10:300:A")
+    assert strict_sorted.index("2:300:A") < strict_sorted.index("10:300:A")
   end
 end

data/test/rbbt/util/misc/test_pipes.rb

@@ -107,6 +107,23 @@ row1 A B C
     assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
   end
 
+  def test_sort_long_stream
+    text =<<-EOF
+##
+##
+##
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row3 AAA BBB CCC
+row1 A B C
+    EOF
+
+    s = StringIO.new text + (text.split("\n")[-3..-1] * "\n" + "\n") * 10000
+    sorted = Misc.sort_stream(s)
+    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
+    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+  end
+
   def test_sort_stream2
     text =<<-EOF
 ##
@@ -318,7 +335,9 @@ line4
 
     TmpFile.with_file do |tmp|
       #Misc.consume_stream(sout, false, tmp)
-      Open.write(tmp, sout)
+      assert_raise do
+        Open.write(tmp, sout)
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.1
+  version: 5.34.4
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-07 00:00:00.000000000 Z
+date: 2022-07-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -164,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: method_source
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Utilities for handling tsv files, caches, etc
 email: miguel.vazquez.g@bsc.es
 executables: