rbbt-util 5.34.2 → 5.34.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cebda38db6f69e3dafffde7a38d5b35c87ead40eee194d4f69563577673a19e8
-  data.tar.gz: 565788a963e37d0850fe14fe9877d644e05d1131d2c56fa27a78fac5e9081efe
+  metadata.gz: '073587b420676db7c055e58dbabaecbf04a7b30ba9d7ea912f1c12df5a6d180d'
+  data.tar.gz: 6d263d07b85095fefb48c15692af0db66b56ef671ada83e98c66a4765d4ca416
 SHA512:
-  metadata.gz: 63dc077571b4c7931ca6a29e880294462c0b1928fce06ae280702f343ba5b51bb383234524e8eb766963a516dd8a5ae8afe83433ac9f56e9e83f20b892cad971
-  data.tar.gz: a956cfeccbb20ced8cbee407159378e8b02490cced6fd56f83b3d62a7bbce9c3f27d0331ed4776fa3c7bf3e945c40e444c4fc409a3965ce2fcaef9c2ff7400d6
+  metadata.gz: 945c0f4b9d6f99024f6610dc2eb0c52c6546606937fff72558281ecf5d2aaddeb2e32ce3373b9348fa764e8e7285cdcdad6916118ba259d61c24b4b3477e3317
+  data.tar.gz: a8ba069b626b1a09d6da8e73eaa901119795ade4b4af33f1891b8581cfe80ccf48ca439a6a96b67f45e6bdf687ba7e90df263b8b4caabec18af45f6008e2b5d9

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2013 Miguel Vázquez García
+Copyright (c) 2010-2022 Miguel Vázquez García
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/rbbt/association/database.rb

@@ -86,6 +86,8 @@ module Association
     info_fields = field_pos.collect{|f| f == :key ? :key : all_fields[f]}
     options = options.merge({:key_field => source_field, :fields =>  info_fields})
 
+    fields = field_headers if fields.nil?
+
     data = options[:data] || {}
     TmpFile.with_file do |tmpfile|
       tmp_data = Persist.open_database(tmpfile, true, :double, "HDB")
@@ -160,6 +162,10 @@ module Association
 
     data ||= {}
     tsv = nil
+    if data.respond_to?(:close) && data.respond_to?(:write)
+      data.close
+      data.write
+    end
     TmpFile.with_file do |tmpfile|
       tmp_data = Persist.open_database(tmpfile, true, open_options[:type], "HDB")
 

data/lib/rbbt/association/open.rb

@@ -24,7 +24,7 @@ module Association
       options = options.dup
       data.serializer = :double if data.respond_to? :serializer
 
-      tsv = Association.database(file, options.merge(:persist => true, :unnamed => true, :data => data, :type => :double))
+      tsv = Association.database(file, options.merge(:unnamed => true, :data => data, :type => :double))
 
       data
     end

data/lib/rbbt/resource/rake.rb

@@ -1,7 +1,7 @@
 require 'rake'
 module Rake
   class TaskNotFound < StandardError; end
-  def self.run(rakefile, dir, task)
+  def self.run(rakefile, dir, task, &block)
     old_pwd = FileUtils.pwd
 
     Rake::FileTask.module_eval do
@@ -32,9 +32,15 @@ module Rake
     t = nil
     pid = Process.fork{
       if block_given?
-        yield
+        TOPLEVEL_BINDING.receiver.instance_exec &block
       else
-        load rakefile
+        if Misc.is_filename? rakefile
+          load rakefile
+        else
+          TmpFile.with_file(rakefile) do |tmpfile|
+            load tmpfile
+          end
+        end
       end
 
       raise TaskNotFound if Rake::Task[task].nil?

data/lib/rbbt/resource/util.rb

@@ -123,8 +123,8 @@ module Resource
 
 
   def rake_for(path)
-    @rake_dirs.reject{|dir, content|
-      !Misc.common_path(dir, path)
+    @rake_dirs.select{|dir, content|
+      Misc.common_path(dir, path)
     }.sort_by{|dir, content|
       dir.length
     }.last
@@ -137,12 +137,17 @@ module Resource
   def run_rake(path, rakefile, rake_dir)
     task = Misc.path_relative_to rake_dir, path
     rakefile = rakefile.produce if rakefile.respond_to? :produce
+    rakefile = rakefile.find if rakefile.respond_to? :find
 
     rake_dir = rake_dir.find(:user) if rake_dir.respond_to? :find
 
     begin
       require 'rbbt/resource/rake'
-      Rake.run(rakefile, rake_dir, task)
+      if Proc === rakefile
+        Rake.run(nil, rake_dir, task, &rakefile)
+      else
+        Rake.run(rakefile, rake_dir, task)
+      end
     rescue Rake::TaskNotFound
       raise $! if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
       task = File.join(File.basename(rake_dir), task)

data/lib/rbbt/resource.rb

@@ -59,7 +59,7 @@ module Resource
 
   def claim(path, type, content = nil, &block)
     if type == :rake
-      @rake_dirs[path] = content
+      @rake_dirs[path] = content || block
     else
       @resources[path] = [type, content || block]
 
@@ -346,9 +346,12 @@ url='#{url}'
     locations = (Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys)
     locations -= [:current, "current"]
     locations << :current
+    search_paths = IndiferentHash.setup(resource.search_paths)
     locations.uniq.each do |name|
-      pattern = resource.search_paths[name]
+      pattern = search_paths[name]
+      pattern = resource.search_paths[pattern] while Symbol === pattern
       next if pattern.nil?
+
       pattern = pattern.sub('{PWD}', Dir.pwd)
       if String ===  pattern and pattern.include?('{')
         regexp = "^" + pattern.gsub(/{([^}]+)}/,'(?<\1>[^/]+)') + "(?:/(?<REST>.*))?/?$"

data/lib/rbbt/tsv/change_id.rb

@@ -89,7 +89,7 @@ module TSV
           tsv[k] = v
         end
       end
-      
+
       tsv.fields = tsv.fields.collect{|f| f == field ? format : f}
     end
 

data/lib/rbbt/tsv/parallel/traverse.rb

@@ -696,7 +696,7 @@ module TSV
 
       options[:join] = Proc.new do |error|
         error = false if error.nil?
-        Log::ProgressBar.remove_bar(bar, error)
+        Log::ProgressBar.remove_bar(bar, error) if bar
       end if bar
 
       options[:callback] = Proc.new do |e|

data/lib/rbbt/tsv/parser.rb

@@ -8,7 +8,9 @@ module TSV
 
     def all_fields
       all = [key_field] + fields
-      NamedArray.setup all, all
+      # ToDo: What was this for?
+      #NamedArray.setup all, all
+      all
     end
 
     def parse_header(stream)
@@ -550,12 +552,12 @@ module TSV
       line = self.rescue_first_line
       line = stream.gets if line.nil?
 
-      if @tsv_grep
+      if @tsv_grep || grep
 
-        stream = Open.grep(stream, @tsv_grep, invert_grep, fixed_grep)
+        stream = Open.grep(stream, @tsv_grep || grep, invert_grep, fixed_grep)
         stream.no_fail = true
         begin
-          match = Open.grep(StringIO.new(line), @tsv_grep, invert_grep, fixed_grep).read
+          match = Open.grep(StringIO.new(line), @tsv_grep || grep, invert_grep, fixed_grep).read
           line = stream.gets if match.empty?
         rescue Exception
           Log.exception $!
@@ -650,7 +652,7 @@ module TSV
           end
         end
       ensure
-        Log::ProgressBar.remove_bar(progress_monitor)
+        Log::ProgressBar.remove_bar(progress_monitor) if progress_monitor
         stream.close unless stream.closed?
         stream.join if stream.respond_to? :join and not stream.joined?
       end

data/lib/rbbt/util/log/progress/report.rb

@@ -65,11 +65,17 @@ module Log
 
       thr = 0.0000001 if thr == 0
       
-      if mean.nil? or mean.to_i > 1
+      if mean.nil? or mean.to_i > 2
         str = "#{ Log.color :blue, thr.to_i.to_s } per sec."
         #str << " #{ Log.color :yellow, mean.to_i.to_s } avg. #{Log.color :yellow, @mean_max.to_i.to_s} max." if @mean_max > 0
       else
-        str = "#{ Log.color :blue, (1/thr).ceil.to_s } secs each"
+        if 1.0/thr < 1
+          str = "#{ Log.color :blue, (1.0/thr).round(2).to_s } secs each"
+        elsif 1.0/thr < 2
+          str = "#{ Log.color :blue, (1.0/thr).round(1).to_s } secs each"
+        else
+          str = "#{ Log.color :blue, (1/thr).ceil.to_s } secs each"
+        end
         #str << " #{ Log.color :yellow, (1/mean).ceil.to_s } avg. #{Log.color :yellow, (1/@mean_max).ceil.to_s} min." if @mean_max > 0
       end
 
@@ -184,6 +190,7 @@ module Log
       @last_time = Time.now
       @last_count = ticks
       @last_percent = percent if max and max > 0
+      Log::LAST.replace "progress"
       save if file
     end
 

data/lib/rbbt/util/misc/exceptions.rb

@@ -1,7 +1,8 @@
 class RbbtException < StandardError; end
 class ParameterException < RbbtException; end
-class FieldNotFoundError < RbbtException;end
-class ClosedStream < RbbtException; end
+
+class FieldNotFoundError < StandardError;end
+class ClosedStream < StandardError; end
 
 class ProcessFailed < StandardError; 
   def initialize(pid = Process.pid)
@@ -26,7 +27,7 @@ end
 class SemaphoreInterrupted < TryAgain; end
 class LockInterrupted < TryAgain; end
 
-class RemoteServerError < RbbtException; end
+class RemoteServerError < StandardError; end
 
 class DependencyError < Aborted
   def initialize(msg)

data/lib/rbbt/util/misc/inspect.rb

@@ -72,7 +72,9 @@ module Misc
     when File
       "<File:" + obj.path + ">"
     when NamedArray
-      "[<NamedArray: fields=#{fingerprint obj.fields} -- values=#{fingerprint obj[0..-1]}]"
+      fields = obj.fields
+      fields = fields.collect if NamedArray === fields
+      "[<NamedArray: fields=#{fingerprint fields} -- values=#{fingerprint obj[0..-1]}]"
     when Array
       if (length = obj.length) > 10
         "[#{length}--" <<  (obj.values_at(0,1, length / 2, -2, -1).collect{|e| fingerprint(e)} * ",") << "]"
@@ -298,6 +300,8 @@ module Misc
     str = case obj
           when nil
             'nil'
+          when Numeric
+            Float === obj && obj % 1 == 0 ? obj.to_i.to_s : obj.to_s
           when Symbol 
             obj.to_s
           when TrueClass

data/lib/rbbt/util/misc/omics.rb

@@ -119,7 +119,7 @@ module Misc
   def self.correct_icgc_mutation(pos, ref, mut_str)
     mut = mut_str
     mut = '-' * (mut_str.length - 1) if mut =~/^-[ACGT]/
-      mut = "+" << mut if ref == '-'
+    mut = "+" << mut if ref == '-'
     [pos, [mut]]
   end
 

data/lib/rbbt/util/misc/pipes.rb

@@ -420,7 +420,7 @@ module Misc
           end
 
           Open.touch path if Open.exists? path
-          content.join if content.respond_to? :join and not (content.respond_to?(:joined?) and content.joined?)
+          content.join if content.respond_to?(:join) and not Path === content and not (content.respond_to?(:joined?) && content.joined?)
 
           Open.notify_write(path) 
         rescue Aborted

data/lib/rbbt/util/named_array.rb

@@ -22,7 +22,7 @@ module NamedArray
   def self.setup(array, fields, key = nil, entity_options = nil, entity_templates = nil)
     return array if array.nil?
     array.extend NamedArray unless NamedArray === array
-    array.fields = Annotated.purge fields
+    array.fields = Annotated === fields ? Annotated.purge(fields) : fields
     array.key = key
     array.entity_options = entity_options unless entity_options.nil?
     array.entity_templates = entity_templates unless entity_templates.nil?

data/lib/rbbt/util/python.rb

@@ -1,9 +1,39 @@
 require 'rbbt-util'
 require 'pycall/import'
+require 'rbbt/util/python/util'
 
 module RbbtPython
   extend PyCall::Import
 
+  def self.script(text, options = {})
+    Log.debug "Running python script:\n#{text.dup}"
+    text = StringIO.new text unless IO === text
+    CMD.cmd_log(:python, options.merge(:in => text))
+  end
+
+  def self.add_path(path)
+    self.run 'sys' do
+      sys.path.append path
+    end
+  end
+
+  def self.add_paths(paths)
+    self.run 'sys' do
+      paths.each do |path|
+        sys.path.append path
+      end
+    end
+  end
+
+  def self.init_rbbt
+    if ! defined?(@@__init_rbbt) || ! @@__init_rbbt 
+      Log.debug "Loading python 'rbbt' module into pycall RbbtPython module"
+      RbbtPython.add_paths(Rbbt.python.find_all)
+      RbbtPython.pyimport("rbbt")
+      @@__init_rbbt = true
+    end
+  end
+
   def self.exec(script)
     PyCall.exec(script)
   end
@@ -63,7 +93,9 @@ module RbbtPython
 
   def self.run_log(mod = nil, imports = nil, severity = 0, severity_err = nil, &block)
     if mod
-      if Array === imports
+      if imports == "*" || imports == ["*"]
+        pyfrom mod
+      elsif Array === imports
         pyfrom mod, :import => imports
       elsif Hash === imports
         pyimport mod, imports
@@ -92,21 +124,4 @@ module RbbtPython
       module_eval(&block)
     end
   end
-
-  def self.add_path(path)
-    self.run 'sys' do
-      sys.path.append path
-    end
-  end
-
-  def self.add_paths(paths)
-    self.run 'sys' do
-      paths.each do |path|
-        sys.path.append path
-      end
-    end
-  end
-
-  RbbtPython.add_paths Rbbt.python.find_all
-  RbbtPython.pyimport "rbbt"
 end

data/lib/rbbt/workflow/definition.rb

@@ -22,8 +22,13 @@ module Workflow
     :extension          => nil)
 
 
-  def helper(name, &block)
-    helpers[name] = block
+  def helper(name, *args, &block)
+    if block_given?
+      helpers[name] = block
+    else
+      raise RbbtException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
+      helpers[name].call(*args)
+    end
   end
 
   def desc(description)

data/lib/rbbt/workflow/dependencies.rb

@@ -71,6 +71,7 @@ module Workflow
 
   def setup_override_dependency(dep, workflow, task_name)
     return [] if dep == :skip || dep == 'skip'
+
     dep = Step === dep ? dep.dup : Workflow.load_step(dep)
 
     dep.original_workflow ||= dep.workflow if dep.workflow

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -148,6 +148,11 @@ job.clean
     def self.upload_inputs(server, inputs, input_types, input_id)
       TmpFile.with_file do |dir|
         if Step.save_inputs(inputs, input_types, dir)
+          Dir.glob(File.join(dir, "*.as_step")).each do |file|
+            path = Open.read(file).strip
+            new = Step.migrate(path, :user, :target => server)
+            Open.write(file, new)
+          end
           CMD.cmd("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
         end
       end

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -3,6 +3,7 @@ class RemoteStep
     attr_accessor :override_dependencies
 
     def init_job(cache_type = nil, other_params = {})
+      return self if @url
       cache_type = :asynchronous if cache_type.nil? and not @is_exec
       cache_type = :exec if cache_type.nil?
       @last_info_time = nil
@@ -10,20 +11,17 @@ class RemoteStep
       @server, @server_path = RemoteWorkflow::SSH.parse_url base_url
       @input_id ||= "inputs-" << rand(100000).to_s
 
-      if override_dependencies
-
-        if override_dependencies && override_dependencies.any?
-          override_dependencies.each do |od|
-            name, _sep, value = od.partition("=")
-            inputs[name] = value
-          end
+      if override_dependencies && override_dependencies.any?
+        override_dependencies.each do |od|
+          name, _sep, value = od.partition("=")
+          inputs[name] = value
         end
-
-        RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
-      else
-        RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
       end
 
+      inputs.select{|i| Step === i }.each{|i| i.produce }
+
+      RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
+
       @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
         Misc.insist do
           input_types = {}

data/lib/rbbt/workflow/step/dependencies.rb

@@ -230,7 +230,14 @@ class Step
       if dep_step[step.path] and dep_step[step.path].length > 1
         stream = step.result
         other_steps = dep_step[step.path].uniq.reject{|d| d.overriden }
+
+        other_steps = other_steps.collect{|d|
+          deps_using_step_input = d.rec_dependencies.select{|d| d.inputs.include? step  }
+          deps_using_step_input.any? ? deps_using_step_input : d
+        }.flatten.uniq
+
         return unless other_steps.length > 1
+
         log_dependency_exec(step, "duplicating #{other_steps.length}") 
         copies = Misc.tee_stream_thread_multiple(stream, other_steps.length)
         copies.extend StreamArray
@@ -370,7 +377,6 @@ class Step
   end
 
   def run_dependencies
-    dep_step = {}
 
     rec_dependencies = self.rec_dependencies(true) + input_dependencies
 
@@ -385,23 +391,30 @@ class Step
 
     canfail_paths = self.canfail_paths
 
+    dep_step = {}
     seen_paths = Set.new
     all_deps.uniq.each do |step|
       next if seen_paths.include? step.path
       seen_paths << step.path
+
       begin
         Step.prepare_for_execution(step) unless step == self
       rescue DependencyError, DependencyRbbtException
         raise $! unless canfail_paths.include? step.path
       end
+
       next unless step.dependencies and step.dependencies.any?
-      (step.dependencies + step.input_dependencies).each do |step_dep|
+
+      # ToDo is this really necessary
+      #(step.dependencies + step.input_dependencies).each do |step_dep|
+      step.dependencies.each do |step_dep|
         next unless step.dependencies.include?(step_dep)
         next if step_dep.done? or step_dep.running? or 
           (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
         dep_step[step_dep.path] ||= []
         dep_step[step_dep.path] << step
       end
+
     end
 
     produced = []

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -51,8 +51,8 @@ module Workflow
         when :path
           inputs[input.to_sym]  = Open.read(file).strip.split("\n").first
         when :io
-          inputs[input.to_sym] = Open.open(Open.realpath(file)).split("\n")
-        when :io
+          inputs[input.to_sym] = Open.open(Open.realpath(file))
+        when :io_array
           inputs[input.to_sym] = Open.realpath(file).split("\n").collect{|f| Open.open(f)}
         when :step_array
           steps = Open.read(file).strip.split("\n").collect{|path| Workflow.load_step(path) }
@@ -97,6 +97,10 @@ module Workflow
           inputs[input.to_sym]  = TSV.open(file)
         when :boolean
           inputs[input.to_sym]  = (file.read.strip == 'true')
+        when :integer
+          inputs[input.to_sym]  = file.read.to_i
+        when :float
+          inputs[input.to_sym]  = file.read.to_f
         else
           Log.debug "Loading #{ input } from #{file}"
           inputs[input.to_sym]  = file.read.strip
@@ -125,86 +129,6 @@ module Workflow
     inputs = task_inputs_from_directory(task_name, directory)
     job(task_name, jobname, inputs)
   end
-
-  #def self.load_inputs_old(dir, input_names, input_types)
-  #  inputs = {}
-  #  if File.exists?(dir) && ! File.directory?(dir)
-  #    Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
-  #    tarfile = dir
-  #    digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
-  #    tmpdir = Rbbt.tmp.input_bundle[digest].find
-  #    Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
-  #    files = tmpdir.glob("*")
-  #    if files.length == 1 && File.directory?(files.first)
-  #      tmpdir = files.first
-  #    end
-  #    load_inputs(tmpdir, input_names, input_types)
-  #  else
-  #    dir = Path.setup(dir.dup)
-  #    input_names.each do |input|
-  #      file = dir[input].find
-  #      file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not (File.symlink?(file) || file.exists?)
-  #      Log.debug "Trying #{ input }: #{file}"
-  #      next unless file and (File.symlink?(file) || file.exists?)
-
-  #      type = input_types[input]
-
-  #      type = :io if file.split(".").last == 'as_io'
-
-  #      type = :path if file.split(".").last == 'as_path'
-
-  #      type = :filename if file.split(".").last == 'as_filename'
-
-  #      type = :nofile if file.split(".").last == 'nofile'
-
-  #      case type
-  #      when :nofile
-  #        inputs[input.to_sym]  = Open.realpath(file)
-  #      when :path
-  #        inputs[input.to_sym]  = Open.realpath(Open.read(file).strip)
-  #      when :io
-  #        inputs[input.to_sym] = Open.open(Open.realpath(file))
-  #      when :file, :binary
-  #        Log.debug "Pointing #{ input } to #{file}"
-  #        if file =~ /\.yaml/
-  #          inputs[input.to_sym]  = YAML.load(Open.read(file))
-  #        else
-  #          if File.symlink?(file)
-  #            link_target = File.expand_path(File.readlink(file), File.dirname(file))
-  #            inputs[input.to_sym]  = link_target
-  #          else
-  #            inputs[input.to_sym]  = Open.realpath(file)
-  #          end
-  #        end
-  #      when :text
-  #        Log.debug "Reading #{ input } from #{file}"
-  #        inputs[input.to_sym]  = Open.read(file)
-  #      when :array
-  #        Log.debug "Reading array #{ input } from #{file}"
-  #        inputs[input.to_sym]  = Open.read(file).split("\n")
-  #      when :tsv
-  #        Log.debug "Opening tsv #{ input } from #{file}"
-  #        inputs[input.to_sym]  = TSV.open(file)
-  #      when :boolean
-  #        inputs[input.to_sym]  = (file.read.strip == 'true')
-  #      else
-  #        Log.debug "Loading #{ input } from #{file}"
-  #        inputs[input.to_sym]  = file.read.strip
-  #      end
-
-  #    end
-  #    inputs = IndiferentHash.setup(inputs)
-
-  #    dir.glob("*#*").each do |od|
-  #      name = File.basename(od)
-  #      value = Open.read(od)
-  #      Log.debug "Loading override dependency #{ name } as #{value}"
-  #      inputs[name] = value.chomp
-  #    end
-
-  #    inputs
-  #  end
-  #end
 end
 
 class Step
@@ -252,10 +176,17 @@ class Step
     end
 
     Log.debug "Saving job input #{name} (#{type}) into #{path}"
-    Open.write(path, value.to_s)
+
+    if value.respond_to? :filename
+      Open.write(path, value.filename)
+    elsif IO === value
+      Open.write(path, value)
+    else
+      Open.write(path, value.to_s)
+    end
   end
 
-  def self.save_inputs(inputs, input_types, input_options, dir)
+  def self.save_inputs(inputs, input_types, dir)
     inputs.each do |name,value|
       type = input_types[name]
       type = type.to_s if type
@@ -300,7 +231,7 @@ class Step
       input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
     end
 
-    save_inputs(inputs, input_types, input_options, dir)
+    save_inputs(inputs, input_types, dir)
 
     inputs.keys
   end

data/lib/rbbt/workflow/step.rb

@@ -363,7 +363,7 @@ class Step
   def recursive_clean
     dependencies.each do |step| 
       step.recursive_clean 
-    end
+    end if dependencies
     clean if Open.exists?(self.info_file)
     self
   end

data/lib/rbbt/workflow/util/archive.rb

@@ -184,9 +184,13 @@ puts files * "\n"
 
     target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
 
+    target_path = File.join(target, *path.split("/")[-3..-1])
+
     subpath_files.each do |subpath, files|
       Rbbt.migrate_files([subpath], target, options.merge(:files => files))
     end
+
+    target_path
   end
 
   def self.purge(path, recursive = false, skip_overriden = true)

data/lib/rbbt/workflow/util/provenance.rb

@@ -109,7 +109,7 @@ class Step
           end
         end
       end
-    end
+    end if step.dependencies
 
     str = ""
     str = " " * offset + this_step_msg if ENV["RBBT_ORIGINAL_STACK"] == 'true'

data/share/Rlib/util.R

@@ -232,20 +232,26 @@ rbbt.tsv2tibble <- function(data){
   as_tibble(data, rownames=attr(data, 'key.field'))
 }
 
-rbbt.tsv.write <- function(filename, data, key.field = NULL, extra_headers = NULL, eol="\n", ...){
+rbbt.tsv.write <- function(filename, data, key.field = NULL, extra_headers = NULL, eol="\n", type = 'list', names = NULL, ...){
 
   if (is.null(key.field)){ key.field = attributes(data)$key.field;}
   if (is.null(key.field)){ key.field = "ID";}
 
   f = file(filename, 'wb');
 
+  if (is.null(extra_headers)){
+    extra_headers = paste(":type",type,sep="=")
+  }
+
   if (!is.null(extra_headers)){
       extra_headers = paste("#: ", extra_headers, "\n", sep="");
       cat(extra_headers, file=f);
   }
 
   header = paste("#", key.field, sep="");
-  for (name in colnames(data)){ header = paste(header, name, sep="\t");}
+  if (is.null(names)){ names = colnames(data)}
+  if (is.null(names)){ names = names(data)}
+  for (name in names){ header = paste(header, name, sep="\t");}
   header = paste(header, "\n", sep="");
   cat(header, file=f);
   

data/share/rbbt_commands/tsv/view

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+View TSV nicer
+
+$ #{$0} [options] <filename.tsv|->
+
+Use - to read from STDIN
+
+-h--help Print this help
+
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+
+file = ARGV.shift
+
+file = STDIN if file == '-'
+
+case
+when options[:tokyocabinet]
+  tsv = Persist.open_tokyocabinet(file, false)
+when options[:tokyocabinet_bd]
+  tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
+else
+  if String === file
+    file = file.dup
+    Path.setup(File.expand_path(file))
+  end
+  tsv = file
+end
+
+tsv = TSV.open(tsv) if IO === tsv
+
+
+begin
+  Workflow.require_workflow "Genomics"
+  tsv = Genomics.job(:names, nil, :tsv => tsv).run
+rescue
+end
+
+puts [Log.highlight(Log.color(:magenta, tsv.key_field)), Log.color(:yellow, tsv.fields * "\t")] * "\t"
+TSV.traverse tsv do |k,values,fields|
+  fvalues = values.zip(fields).collect do |v,field|
+    if String === v
+      v = v.to_f.round(2) if v =~ /^-?\d+\.\d+$/
+      v = v.to_i if v =~ /^-?\d+$/
+    end
+
+    if Numeric === v
+      if field =~ /p.?value/i
+        v = v.round(4)
+        v < 0.05 ? Log.color(:cyan, v.to_s) : v
+      else
+        v = v.round(2)
+        v < 0 ? Log.color(:red, v.to_s) : Log.color(:green, v.to_s)
+      end
+    else
+      v
+    end
+  end
+  puts [Log.color(:magenta, k), fvalues] * "\t"
+end

data/share/rbbt_commands/workflow/server

@@ -114,7 +114,7 @@ TmpFile.with_file do |app_dir|
         name, _sep, value = pair.partition("=")
         name = name[1..-1].to_sym if name[0] == ':'
         value = value.to_i if value =~ /^\d+$/
-        value = true if value == "true"
+        value = true if value.nil? || value == "true"
         value = false if value == "false"
         options[name] = value
       end

data/share/rbbt_commands/workflow/task

@@ -598,14 +598,14 @@ when Step
   elsif detach
     exit! 0
   else
-    res.join
+      res.join if res.running?
     if %w(float integer string boolean).include?(res.result_type.to_s)
       out.puts res.load
     else
       Open.open(res.path, :mode => 'rb') do |io|
         Misc.consume_stream(io, false, out)
       end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
-    end
+    end if res.done?
   end
 else
   if Array === res

data/test/rbbt/test_resource.rb

@@ -17,9 +17,18 @@ module TestResource
 file 'foo' do |t|
   Open.write(t.name, "TEST")
 end
+
+rule /.*/ do |t|
+  Open.write(t.name, "bar")
+end
   EOF
 
-  claim tmp.test.work.footest, :rake, tmp.test.rakefiles.foo
+  claim tmp.test.work.footest, :rake, TestResource.tmp.test.rakefiles.foo
+
+  claim tmp.test.work.file_proc, :file_proc do |file,filename|
+    Open.write(filename, file)
+    nil
+  end
 end
 
 class TestTSV < Test::Unit::TestCase
@@ -33,11 +42,12 @@ class TestTSV < Test::Unit::TestCase
     assert TSV === TestResource.tmp.test.test_tsv.tsv
   end
 
-  def __test_rake
-    TestResource.tmp.test.work.footest.foo.read == "TEST"
-    assert TestResource.tmp.test.work.footest.foo.read == "TEST"
+  def test_rake
+    assert_equal TestResource.tmp.test.work.footest.foo.read, "TEST"
+    assert_equal TestResource.tmp.test.work.footest.bar.read, "bar"
   end
 
+
   def test_proc
     assert TestResource.tmp.test.proc.read == "PROC TEST"
   end

data/test/rbbt/util/misc/test_omics.rb

@@ -88,8 +88,28 @@ class TestMiscOmics < Test::Unit::TestCase
       index = Misc.index_BED(io, dir)
       assert_equal ["2:2"], index["2:220:230"]
     end
+  end
 
+  def test_sort_genomic_locations
+    mutations =<<-EOF.split("\n").shuffle
+1:100:A
+1:20:A
+1:300:A
+2:100:A
+2:20:A
+2:300:A
+10:100:A
+10:20:A
+10:300:A
+    EOF
+    sorted =  Misc.sort_mutation_stream(StringIO.new(mutations * "\n")).read.split("\n")
+    strict_sorted = Misc.sort_mutation_stream_strict(StringIO.new(mutations * "\n")).read.split("\n")
 
-
+    assert sorted.index("1:20:A") < sorted.index("1:100:A")
+    assert sorted.index("1:300:A") < sorted.index("10:300:A")
+    assert sorted.index("10:300:A") < sorted.index("2:300:A")
+    assert strict_sorted.index("1:20:A") < strict_sorted.index("1:100:A")
+    assert strict_sorted.index("1:300:A") < strict_sorted.index("10:300:A")
+    assert strict_sorted.index("2:300:A") < strict_sorted.index("10:300:A")
   end
 end

data/test/rbbt/util/misc/test_pipes.rb

@@ -107,6 +107,23 @@ row1 A B C
     assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
   end
 
+  def test_sort_long_stream
+    text =<<-EOF
+##
+##
+##
+#Row LabelA LabelB LabelC
+row2 AA BB CC
+row3 AAA BBB CCC
+row1 A B C
+    EOF
+
+    s = StringIO.new text + (text.split("\n")[-3..-1] * "\n" + "\n") * 10000
+    sorted = Misc.sort_stream(s)
+    assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
+    assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
+  end
+
   def test_sort_stream2
     text =<<-EOF
 ##
@@ -318,7 +335,9 @@ line4
 
     TmpFile.with_file do |tmp|
       #Misc.consume_stream(sout, false, tmp)
-      Open.write(tmp, sout)
+      assert_raise do
+        Open.write(tmp, sout)
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.2
+  version: 5.34.5
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-06-13 00:00:00.000000000 Z
+date: 2022-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -164,6 +164,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: method_source
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Utilities for handling tsv files, caches, etc
 email: miguel.vazquez.g@bsc.es
 executables:
@@ -445,6 +459,7 @@ files:
 - share/rbbt_commands/tsv/uncollapse
 - share/rbbt_commands/tsv/unzip
 - share/rbbt_commands/tsv/values
+- share/rbbt_commands/tsv/view
 - share/rbbt_commands/tsv/write_excel
 - share/rbbt_commands/tsv/zip
 - share/rbbt_commands/watch