rbbt-util 5.34.2 → 5.34.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cebda38db6f69e3dafffde7a38d5b35c87ead40eee194d4f69563577673a19e8
-  data.tar.gz: 565788a963e37d0850fe14fe9877d644e05d1131d2c56fa27a78fac5e9081efe
+  metadata.gz: ac036317d84e57e31c930adcd9f6f3b83951fd858772a59d04045f6b83c747f1
+  data.tar.gz: 072e7053018bb6d38b7db1ed3164f5fe89e534264918bf1657154ef6d689f5f1
 SHA512:
-  metadata.gz: 63dc077571b4c7931ca6a29e880294462c0b1928fce06ae280702f343ba5b51bb383234524e8eb766963a516dd8a5ae8afe83433ac9f56e9e83f20b892cad971
-  data.tar.gz: a956cfeccbb20ced8cbee407159378e8b02490cced6fd56f83b3d62a7bbce9c3f27d0331ed4776fa3c7bf3e945c40e444c4fc409a3965ce2fcaef9c2ff7400d6
+  metadata.gz: e399e00198996cfc6958e897b41359824c3277511a9ddd402cb46a8f7cebbe99ce560ffa3f645afb7e64983f1fb1c31cb1d0269adde2333e3f43f5764b9a5393
+  data.tar.gz: 3341baac06e6aa8f2517d1bc812a89f3ac86d30f13483c394b7557bb20fcc36ca4846402d18e83587ee6329047bc9445892e99cf4a94d48fed9555ebef2b4804

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2013 Miguel Vázquez García
+Copyright (c) 2010-2022 Miguel Vázquez García
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/rbbt/association/database.rb

@@ -86,6 +86,8 @@ module Association
     info_fields = field_pos.collect{|f| f == :key ? :key : all_fields[f]}
     options = options.merge({:key_field => source_field, :fields =>  info_fields})
 
+    fields = field_headers if fields.nil?
+
     data = options[:data] || {}
     TmpFile.with_file do |tmpfile|
       tmp_data = Persist.open_database(tmpfile, true, :double, "HDB")

data/lib/rbbt/association/open.rb

@@ -24,7 +24,7 @@ module Association
       options = options.dup
       data.serializer = :double if data.respond_to? :serializer
 
-      tsv = Association.database(file, options.merge(:persist => true, :unnamed => true, :data => data, :type => :double))
+      tsv = Association.database(file, options.merge(:unnamed => true, :data => data, :type => :double))
 
       data
     end

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -696,7 +696,7 @@ module TSV
 
       options[:join] = Proc.new do |error|
         error = false if error.nil?
-        Log::ProgressBar.remove_bar(bar, error)
+        Log::ProgressBar.remove_bar(bar, error) if bar
       end if bar
 
       options[:callback] = Proc.new do |e|

data/lib/rbbt/tsv/parser.rb

@@ -650,7 +650,7 @@ module TSV
           end
         end
       ensure
-        Log::ProgressBar.remove_bar(progress_monitor)
+        Log::ProgressBar.remove_bar(progress_monitor) if progress_monitor
         stream.close unless stream.closed?
         stream.join if stream.respond_to? :join and not stream.joined?
       end

data/lib/rbbt/util/log/progress/report.rb

@@ -65,11 +65,17 @@ module Log
 
       thr = 0.0000001 if thr == 0
       
-      if mean.nil? or mean.to_i > 1
+      if mean.nil? or mean.to_i > 2
         str = "#{ Log.color :blue, thr.to_i.to_s } per sec."
         #str << " #{ Log.color :yellow, mean.to_i.to_s } avg. #{Log.color :yellow, @mean_max.to_i.to_s} max." if @mean_max > 0
       else
-        str = "#{ Log.color :blue, (1/thr).ceil.to_s } secs each"
+        if 1.0/thr < 1
+          str = "#{ Log.color :blue, (1.0/thr).round(2).to_s } secs each"
+        elsif 1.0/thr < 2
+          str = "#{ Log.color :blue, (1.0/thr).round(1).to_s } secs each"
+        else
+          str = "#{ Log.color :blue, (1/thr).ceil.to_s } secs each"
+        end
         #str << " #{ Log.color :yellow, (1/mean).ceil.to_s } avg. #{Log.color :yellow, (1/@mean_max).ceil.to_s} min." if @mean_max > 0
       end
 
@@ -184,6 +190,7 @@ module Log
       @last_time = Time.now
       @last_count = ticks
       @last_percent = percent if max and max > 0
+      Log::LAST.replace "progress"
       save if file
     end
 

data/lib/rbbt/util/misc/exceptions.rb

@@ -1,7 +1,8 @@
 class RbbtException < StandardError; end
 class ParameterException < RbbtException; end
-class FieldNotFoundError < RbbtException;end
-class ClosedStream < RbbtException; end
+
+class FieldNotFoundError < StandardError;end
+class ClosedStream < StandardError; end
 
 class ProcessFailed < StandardError; 
   def initialize(pid = Process.pid)
@@ -26,7 +27,7 @@ end
 class SemaphoreInterrupted < TryAgain; end
 class LockInterrupted < TryAgain; end
 
-class RemoteServerError < RbbtException; end
+class RemoteServerError < StandardError; end
 
 class DependencyError < Aborted
   def initialize(msg)

data/lib/rbbt/util/misc/inspect.rb

@@ -298,6 +298,8 @@ module Misc
     str = case obj
           when nil
             'nil'
+          when Numeric
+            obj.to_f
           when Symbol 
             obj.to_s
           when TrueClass

data/lib/rbbt/util/named_array.rb

@@ -22,7 +22,7 @@ module NamedArray
   def self.setup(array, fields, key = nil, entity_options = nil, entity_templates = nil)
     return array if array.nil?
     array.extend NamedArray unless NamedArray === array
-    array.fields = Annotated.purge fields
+    array.fields = Annotated === fields ? Annotated.purge(fields) : fields
     array.key = key
     array.entity_options = entity_options unless entity_options.nil?
     array.entity_templates = entity_templates unless entity_templates.nil?

data/lib/rbbt/util/python.rb

@@ -4,6 +4,30 @@ require 'pycall/import'
 module RbbtPython
   extend PyCall::Import
 
+
+  def self.add_path(path)
+    self.run 'sys' do
+      sys.path.append path
+    end
+  end
+
+  def self.add_paths(paths)
+    self.run 'sys' do
+      paths.each do |path|
+        sys.path.append path
+      end
+    end
+  end
+
+  def self.init_rbbt
+    if ! defined?(@@__init_rbbt) || ! @@__init_rbbt 
+      Log.debug "Loading python 'rbbt' module into pycall RbbtPython module"
+      RbbtPython.add_paths(Rbbt.python.find_all)
+      RbbtPython.pyimport("rbbt")
+      @@__init_rbbt = true
+    end
+  end
+
   def self.exec(script)
     PyCall.exec(script)
   end
@@ -92,21 +116,4 @@ module RbbtPython
       module_eval(&block)
     end
   end
-
-  def self.add_path(path)
-    self.run 'sys' do
-      sys.path.append path
-    end
-  end
-
-  def self.add_paths(paths)
-    self.run 'sys' do
-      paths.each do |path|
-        sys.path.append path
-      end
-    end
-  end
-
-  RbbtPython.add_paths Rbbt.python.find_all
-  RbbtPython.pyimport "rbbt"
 end

data/lib/rbbt/workflow/dependencies.rb

@@ -71,6 +71,7 @@ module Workflow
 
   def setup_override_dependency(dep, workflow, task_name)
     return [] if dep == :skip || dep == 'skip'
+
     dep = Step === dep ? dep.dup : Workflow.load_step(dep)
 
     dep.original_workflow ||= dep.workflow if dep.workflow

data/lib/rbbt/workflow/step/dependencies.rb

@@ -230,7 +230,14 @@ class Step
       if dep_step[step.path] and dep_step[step.path].length > 1
         stream = step.result
         other_steps = dep_step[step.path].uniq.reject{|d| d.overriden }
+
+        other_steps = other_steps.collect{|d|
+          deps_using_step_input = d.rec_dependencies.select{|d| d.inputs.include? step  }
+          deps_using_step_input.any? ? deps_using_step_input : d
+        }.flatten.uniq
+
         return unless other_steps.length > 1
+
         log_dependency_exec(step, "duplicating #{other_steps.length}") 
         copies = Misc.tee_stream_thread_multiple(stream, other_steps.length)
         copies.extend StreamArray
@@ -370,7 +377,6 @@ class Step
   end
 
   def run_dependencies
-    dep_step = {}
 
     rec_dependencies = self.rec_dependencies(true) + input_dependencies
 
@@ -385,23 +391,30 @@ class Step
 
     canfail_paths = self.canfail_paths
 
+    dep_step = {}
     seen_paths = Set.new
     all_deps.uniq.each do |step|
       next if seen_paths.include? step.path
       seen_paths << step.path
+
       begin
         Step.prepare_for_execution(step) unless step == self
       rescue DependencyError, DependencyRbbtException
         raise $! unless canfail_paths.include? step.path
       end
+
       next unless step.dependencies and step.dependencies.any?
-      (step.dependencies + step.input_dependencies).each do |step_dep|
+
+      # ToDo is this really necessary
+      #(step.dependencies + step.input_dependencies).each do |step_dep|
+      step.dependencies.each do |step_dep|
         next unless step.dependencies.include?(step_dep)
         next if step_dep.done? or step_dep.running? or 
           (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
         dep_step[step_dep.path] << step
       end
+
     end
 
     produced = []

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -125,86 +125,6 @@ module Workflow
     inputs = task_inputs_from_directory(task_name, directory)
     job(task_name, jobname, inputs)
   end
-
-  #def self.load_inputs_old(dir, input_names, input_types)
-  #  inputs = {}
-  #  if File.exists?(dir) && ! File.directory?(dir)
-  #    Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
-  #    tarfile = dir
-  #    digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
-  #    tmpdir = Rbbt.tmp.input_bundle[digest].find
-  #    Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
-  #    files = tmpdir.glob("*")
-  #    if files.length == 1 && File.directory?(files.first)
-  #      tmpdir = files.first
-  #    end
-  #    load_inputs(tmpdir, input_names, input_types)
-  #  else
-  #    dir = Path.setup(dir.dup)
-  #    input_names.each do |input|
-  #      file = dir[input].find
-  #      file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not (File.symlink?(file) || file.exists?)
-  #      Log.debug "Trying #{ input }: #{file}"
-  #      next unless file and (File.symlink?(file) || file.exists?)
-
-  #      type = input_types[input]
-
-  #      type = :io if file.split(".").last == 'as_io'
-
-  #      type = :path if file.split(".").last == 'as_path'
-
-  #      type = :filename if file.split(".").last == 'as_filename'
-
-  #      type = :nofile if file.split(".").last == 'nofile'
-
-  #      case type
-  #      when :nofile
-  #        inputs[input.to_sym]  = Open.realpath(file)
-  #      when :path
-  #        inputs[input.to_sym]  = Open.realpath(Open.read(file).strip)
-  #      when :io
-  #        inputs[input.to_sym] = Open.open(Open.realpath(file))
-  #      when :file, :binary
-  #        Log.debug "Pointing #{ input } to #{file}"
-  #        if file =~ /\.yaml/
-  #          inputs[input.to_sym]  = YAML.load(Open.read(file))
-  #        else
-  #          if File.symlink?(file)
-  #            link_target = File.expand_path(File.readlink(file), File.dirname(file))
-  #            inputs[input.to_sym]  = link_target
-  #          else
-  #            inputs[input.to_sym]  = Open.realpath(file)
-  #          end
-  #        end
-  #      when :text
-  #        Log.debug "Reading #{ input } from #{file}"
-  #        inputs[input.to_sym]  = Open.read(file)
-  #      when :array
-  #        Log.debug "Reading array #{ input } from #{file}"
-  #        inputs[input.to_sym]  = Open.read(file).split("\n")
-  #      when :tsv
-  #        Log.debug "Opening tsv #{ input } from #{file}"
-  #        inputs[input.to_sym]  = TSV.open(file)
-  #      when :boolean
-  #        inputs[input.to_sym]  = (file.read.strip == 'true')
-  #      else
-  #        Log.debug "Loading #{ input } from #{file}"
-  #        inputs[input.to_sym]  = file.read.strip
-  #      end
-
-  #    end
-  #    inputs = IndiferentHash.setup(inputs)
-
-  #    dir.glob("*#*").each do |od|
-  #      name = File.basename(od)
-  #      value = Open.read(od)
-  #      Log.debug "Loading override dependency #{ name } as #{value}"
-  #      inputs[name] = value.chomp
-  #    end
-
-  #    inputs
-  #  end
-  #end
 end
 
 class Step

data/share/rbbt_commands/workflow/server

@@ -114,7 +114,7 @@ TmpFile.with_file do |app_dir|
         name, _sep, value = pair.partition("=")
         name = name[1..-1].to_sym if name[0] == ':'
         value = value.to_i if value =~ /^\d+$/
-        value = true if value == "true"
+        value = true if value.nil? || value == "true"
         value = false if value == "false"
         options[name] = value
       end

data/share/rbbt_commands/workflow/task

@@ -598,14 +598,14 @@ when Step
   elsif detach
     exit! 0
   else
-    res.join
+      res.join if res.running?
     if %w(float integer string boolean).include?(res.result_type.to_s)
       out.puts res.load
     else
       Open.open(res.path, :mode => 'rb') do |io|
         Misc.consume_stream(io, false, out)
       end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
-    end
+    end if res.done?
   end
 else
   if Array === res

data/test/rbbt/util/misc/test_omics.rb

@@ -88,8 +88,28 @@ class TestMiscOmics < Test::Unit::TestCase
       index = Misc.index_BED(io, dir)
       assert_equal ["2:2"], index["2:220:230"]
     end
+  end
 
+  def test_sort_genomic_locations
+    mutations =<<-EOF.split("\n").shuffle
+1:100:A
+1:20:A
+1:300:A
+2:100:A
+2:20:A
+2:300:A
+10:100:A
+10:20:A
+10:300:A
+    EOF
+    sorted =  Misc.sort_mutation_stream(StringIO.new(mutations * "\n")).read.split("\n")
+    strict_sorted = Misc.sort_mutation_stream_strict(StringIO.new(mutations * "\n")).read.split("\n")
 
-
+    assert sorted.index("1:20:A") < sorted.index("1:100:A")
+    assert sorted.index("1:300:A") < sorted.index("10:300:A")
+    assert sorted.index("10:300:A") < sorted.index("2:300:A")
+    assert strict_sorted.index("1:20:A") < strict_sorted.index("1:100:A")
+    assert strict_sorted.index("1:300:A") < strict_sorted.index("10:300:A")
+    assert strict_sorted.index("2:300:A") < strict_sorted.index("10:300:A")
   end
 end

data/test/rbbt/util/misc/test_pipes.rb

@@ -107,6 +107,23 @@ row1 A B C
     assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
   end
 
+  def test_sort_long_stream
+    text =<<-EOF
+##
+##
+##
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row3 AAA BBB CCC
+row1 A B C
+    EOF
+
+    s = StringIO.new text + (text.split("\n")[-3..-1] * "\n" + "\n") * 10000
+    sorted = Misc.sort_stream(s)
+    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
+    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+  end
+
   def test_sort_stream2
     text =<<-EOF
 ##
@@ -318,7 +335,9 @@ line4
 
     TmpFile.with_file do |tmp|
       #Misc.consume_stream(sout, false, tmp)
-      Open.write(tmp, sout)
+      assert_raise do
+        Open.write(tmp, sout)
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.2
+  version: 5.34.3
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-13 00:00:00.000000000 Z
+date: 2022-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -164,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: method_source
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Utilities for handling tsv files, caches, etc
 email: miguel.vazquez.g@bsc.es
 executables: