rbbt-util 5.34.4 → 5.34.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eda1db070a45b3361a0c5c6b02fd59e39b031ae4264771f015e4068a888d1f09
-  data.tar.gz: 0f9a4bfafce4180f1db402510f83a880e22f5a75d92bf7506781801ce61575a2
+  metadata.gz: '073587b420676db7c055e58dbabaecbf04a7b30ba9d7ea912f1c12df5a6d180d'
+  data.tar.gz: 6d263d07b85095fefb48c15692af0db66b56ef671ada83e98c66a4765d4ca416
 SHA512:
-  metadata.gz: 7738b49efed37124e80b5aa72b367e9d92d32dc13d64fceef5c452e5b50687882d5bc03106aa906c7f425937ceac72694f8eb8959e2c9d3402164b8618c4c639
-  data.tar.gz: de617b931b585915e029f8f23124b6ce5cccc9091dec25e2fc3fdd8c4d2eaf4e23bfce798c73b753cb08f590933c316ceb7deffafd8e8aca854ea175d5744014
+  metadata.gz: 945c0f4b9d6f99024f6610dc2eb0c52c6546606937fff72558281ecf5d2aaddeb2e32ce3373b9348fa764e8e7285cdcdad6916118ba259d61c24b4b3477e3317
+  data.tar.gz: a8ba069b626b1a09d6da8e73eaa901119795ade4b4af33f1891b8581cfe80ccf48ca439a6a96b67f45e6bdf687ba7e90df263b8b4caabec18af45f6008e2b5d9

data/lib/rbbt/association/database.rb

@@ -162,6 +162,10 @@ module Association
 
     data ||= {}
     tsv = nil
+    if data.respond_to?(:close) && data.respond_to?(:write)
+      data.close
+      data.write
+    end
     TmpFile.with_file do |tmpfile|
       tmp_data = Persist.open_database(tmpfile, true, open_options[:type], "HDB")
 

data/lib/rbbt/resource/rake.rb

@@ -1,7 +1,7 @@
 require 'rake'
 module Rake
   class TaskNotFound < StandardError; end
-  def self.run(rakefile, dir, task)
+  def self.run(rakefile, dir, task, &block)
     old_pwd = FileUtils.pwd
 
     Rake::FileTask.module_eval do
@@ -32,9 +32,15 @@ module Rake
     t = nil
     pid = Process.fork{
       if block_given?
-        yield
+        TOPLEVEL_BINDING.receiver.instance_exec &block
       else
-        load rakefile
+        if Misc.is_filename? rakefile
+          load rakefile
+        else
+          TmpFile.with_file(rakefile) do |tmpfile|
+            load tmpfile
+          end
+        end
       end
 
       raise TaskNotFound if Rake::Task[task].nil?

data/lib/rbbt/resource/util.rb

@@ -123,8 +123,8 @@ module Resource
 
 
   def rake_for(path)
-    @rake_dirs.reject{|dir, content|
-      !Misc.common_path(dir, path)
+    @rake_dirs.select{|dir, content|
+      Misc.common_path(dir, path)
     }.sort_by{|dir, content|
       dir.length
     }.last
@@ -137,12 +137,17 @@ module Resource
   def run_rake(path, rakefile, rake_dir)
     task = Misc.path_relative_to rake_dir, path
     rakefile = rakefile.produce if rakefile.respond_to? :produce
+    rakefile = rakefile.find if rakefile.respond_to? :find
 
     rake_dir = rake_dir.find(:user) if rake_dir.respond_to? :find
 
     begin
       require 'rbbt/resource/rake'
-      Rake.run(rakefile, rake_dir, task)
+      if Proc === rakefile
+        Rake.run(nil, rake_dir, task, &rakefile)
+      else
+        Rake.run(rakefile, rake_dir, task)
+      end
     rescue Rake::TaskNotFound
       raise $! if rake_dir.nil? or rake_dir.empty? or rake_dir == "/" or rake_dir == "./"
       task = File.join(File.basename(rake_dir), task)

data/lib/rbbt/resource.rb

@@ -59,7 +59,7 @@ module Resource
 
   def claim(path, type, content = nil, &block)
     if type == :rake
-      @rake_dirs[path] = content
+      @rake_dirs[path] = content || block
     else
       @resources[path] = [type, content || block]
 

data/lib/rbbt/tsv/change_id.rb

@@ -89,7 +89,7 @@ module TSV
           tsv[k] = v
         end
       end
-      
+
       tsv.fields = tsv.fields.collect{|f| f == field ? format : f}
     end
 

data/lib/rbbt/tsv/parser.rb

@@ -8,7 +8,9 @@ module TSV
 
     def all_fields
       all = [key_field] + fields
-      NamedArray.setup all, all
+      # ToDo: What was this for?
+      #NamedArray.setup all, all
+      all
     end
 
     def parse_header(stream)
@@ -550,12 +552,12 @@ module TSV
       line = self.rescue_first_line
       line = stream.gets if line.nil?
 
-      if @tsv_grep
+      if @tsv_grep || grep
 
-        stream = Open.grep(stream, @tsv_grep, invert_grep, fixed_grep)
+        stream = Open.grep(stream, @tsv_grep || grep, invert_grep, fixed_grep)
         stream.no_fail = true
         begin
-          match = Open.grep(StringIO.new(line), @tsv_grep, invert_grep, fixed_grep).read
+          match = Open.grep(StringIO.new(line), @tsv_grep || grep, invert_grep, fixed_grep).read
           line = stream.gets if match.empty?
         rescue Exception
           Log.exception $!

data/lib/rbbt/util/misc/inspect.rb

@@ -72,7 +72,9 @@ module Misc
     when File
       "<File:" + obj.path + ">"
     when NamedArray
-      "[<NamedArray: fields=#{fingerprint obj.fields} -- values=#{fingerprint obj[0..-1]}]"
+      fields = obj.fields
+      fields = fields.collect if NamedArray === fields
+      "[<NamedArray: fields=#{fingerprint fields} -- values=#{fingerprint obj[0..-1]}]"
     when Array
       if (length = obj.length) > 10
         "[#{length}--" <<  (obj.values_at(0,1, length / 2, -2, -1).collect{|e| fingerprint(e)} * ",") << "]"
@@ -299,7 +301,7 @@ module Misc
           when nil
             'nil'
           when Numeric
-            obj.to_f
+            Float === obj && obj % 1 == 0 ? obj.to_i.to_s : obj.to_s
           when Symbol 
             obj.to_s
           when TrueClass

data/lib/rbbt/util/misc/omics.rb

@@ -119,7 +119,7 @@ module Misc
   def self.correct_icgc_mutation(pos, ref, mut_str)
     mut = mut_str
     mut = '-' * (mut_str.length - 1) if mut =~/^-[ACGT]/
-      mut = "+" << mut if ref == '-'
+    mut = "+" << mut if ref == '-'
     [pos, [mut]]
   end
 

data/lib/rbbt/util/python.rb

@@ -1,5 +1,6 @@
 require 'rbbt-util'
 require 'pycall/import'
+require 'rbbt/util/python/util'
 
 module RbbtPython
   extend PyCall::Import

data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb

@@ -148,6 +148,11 @@ job.clean
     def self.upload_inputs(server, inputs, input_types, input_id)
       TmpFile.with_file do |dir|
         if Step.save_inputs(inputs, input_types, dir)
+          Dir.glob(File.join(dir, "*.as_step")).each do |file|
+            path = Open.read(file).strip
+            new = Step.migrate(path, :user, :target => server)
+            Open.write(file, new)
+          end
           CMD.cmd("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
         end
       end

data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb

@@ -3,6 +3,7 @@ class RemoteStep
     attr_accessor :override_dependencies
 
     def init_job(cache_type = nil, other_params = {})
+      return self if @url
       cache_type = :asynchronous if cache_type.nil? and not @is_exec
       cache_type = :exec if cache_type.nil?
       @last_info_time = nil
@@ -10,20 +11,17 @@ class RemoteStep
       @server, @server_path = RemoteWorkflow::SSH.parse_url base_url
       @input_id ||= "inputs-" << rand(100000).to_s
 
-      if override_dependencies
-
-        if override_dependencies && override_dependencies.any?
-          override_dependencies.each do |od|
-            name, _sep, value = od.partition("=")
-            inputs[name] = value
-          end
+      if override_dependencies && override_dependencies.any?
+        override_dependencies.each do |od|
+          name, _sep, value = od.partition("=")
+          inputs[name] = value
         end
-
-        RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
-      else
-        RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
       end
 
+      inputs.select{|i| Step === i }.each{|i| i.produce }
+
+      RemoteWorkflow::SSH.upload_inputs(@server, inputs, @input_types, @input_id)
+
       @name ||= Persist.memory("RemoteSteps", :workflow => self, :task => task, :jobname => @name, :inputs => inputs, :cache_type => cache_type) do
         Misc.insist do
           input_types = {}

data/lib/rbbt/workflow/step/save_load_inputs.rb

@@ -51,8 +51,8 @@ module Workflow
         when :path
           inputs[input.to_sym]  = Open.read(file).strip.split("\n").first
         when :io
-          inputs[input.to_sym] = Open.open(Open.realpath(file)).split("\n")
-        when :io
+          inputs[input.to_sym] = Open.open(Open.realpath(file))
+        when :io_array
           inputs[input.to_sym] = Open.realpath(file).split("\n").collect{|f| Open.open(f)}
         when :step_array
           steps = Open.read(file).strip.split("\n").collect{|path| Workflow.load_step(path) }
@@ -97,6 +97,10 @@ module Workflow
           inputs[input.to_sym]  = TSV.open(file)
         when :boolean
           inputs[input.to_sym]  = (file.read.strip == 'true')
+        when :integer
+          inputs[input.to_sym]  = file.read.to_i
+        when :float
+          inputs[input.to_sym]  = file.read.to_f
         else
           Log.debug "Loading #{ input } from #{file}"
           inputs[input.to_sym]  = file.read.strip
@@ -172,10 +176,17 @@ class Step
     end
 
     Log.debug "Saving job input #{name} (#{type}) into #{path}"
-    Open.write(path, value.to_s)
+
+    if value.respond_to? :filename
+      Open.write(path, value.filename)
+    elsif IO === value
+      Open.write(path, value)
+    else
+      Open.write(path, value.to_s)
+    end
   end
 
-  def self.save_inputs(inputs, input_types, input_options, dir)
+  def self.save_inputs(inputs, input_types, dir)
     inputs.each do |name,value|
       type = input_types[name]
       type = type.to_s if type
@@ -220,7 +231,7 @@ class Step
       input_types = IndiferentHash.setup(input_types.merge(:override_dependencies => :array))
     end
 
-    save_inputs(inputs, input_types, input_options, dir)
+    save_inputs(inputs, input_types, dir)
 
     inputs.keys
   end

data/lib/rbbt/workflow/step.rb

@@ -363,7 +363,7 @@ class Step
   def recursive_clean
     dependencies.each do |step| 
       step.recursive_clean 
-    end
+    end if dependencies
     clean if Open.exists?(self.info_file)
     self
   end

data/lib/rbbt/workflow/util/archive.rb

@@ -184,9 +184,13 @@ puts files * "\n"
 
     target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
 
+    target_path = File.join(target, *path.split("/")[-3..-1])
+
     subpath_files.each do |subpath, files|
       Rbbt.migrate_files([subpath], target, options.merge(:files => files))
     end
+
+    target_path
   end
 
   def self.purge(path, recursive = false, skip_overriden = true)

data/lib/rbbt/workflow/util/provenance.rb

@@ -109,7 +109,7 @@ class Step
           end
         end
       end
-    end
+    end if step.dependencies
 
     str = ""
     str = " " * offset + this_step_msg if ENV["RBBT_ORIGINAL_STACK"] == 'true'

data/share/Rlib/util.R

@@ -232,20 +232,26 @@ rbbt.tsv2tibble <- function(data){
   as_tibble(data, rownames=attr(data, 'key.field'))
 }
 
-rbbt.tsv.write <- function(filename, data, key.field = NULL, extra_headers = NULL, eol="\n", ...){
+rbbt.tsv.write <- function(filename, data, key.field = NULL, extra_headers = NULL, eol="\n", type = 'list', names = NULL, ...){
 
   if (is.null(key.field)){ key.field = attributes(data)$key.field;}
   if (is.null(key.field)){ key.field = "ID";}
 
   f = file(filename, 'wb');
 
+  if (is.null(extra_headers)){
+    extra_headers = paste(":type",type,sep="=")
+  }
+
   if (!is.null(extra_headers)){
       extra_headers = paste("#: ", extra_headers, "\n", sep="");
       cat(extra_headers, file=f);
   }
 
   header = paste("#", key.field, sep="");
-  for (name in colnames(data)){ header = paste(header, name, sep="\t");}
+  if (is.null(names)){ names = colnames(data)}
+  if (is.null(names)){ names = names(data)}
+  for (name in names){ header = paste(header, name, sep="\t");}
   header = paste(header, "\n", sep="");
   cat(header, file=f);
   

data/share/rbbt_commands/tsv/view

@@ -0,0 +1,76 @@
+#!/usr/bin/env ruby
+
+require 'rbbt-util'
+require 'rbbt/util/simpleopt'
+
+$0 = "rbbt #{$previous_commands*" "} #{ File.basename(__FILE__) }" if $previous_commands
+
+options = SOPT.setup <<EOF
+
+View TSV nicer
+
+$ #{$0} [options] <filename.tsv|->
+
+Use - to read from STDIN
+
+-h--help Print this help
+
+EOF
+if options[:help]
+  if defined? rbbt_usage
+    rbbt_usage 
+  else
+    puts SOPT.doc
+  end
+  exit 0
+end
+
+
+file = ARGV.shift
+
+file = STDIN if file == '-'
+
+case
+when options[:tokyocabinet]
+  tsv = Persist.open_tokyocabinet(file, false)
+when options[:tokyocabinet_bd]
+  tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
+else
+  if String === file
+    file = file.dup
+    Path.setup(File.expand_path(file))
+  end
+  tsv = file
+end
+
+tsv = TSV.open(tsv) if IO === tsv
+
+
+begin
+  Workflow.require_workflow "Genomics"
+  tsv = Genomics.job(:names, nil, :tsv => tsv).run
+rescue
+end
+
+puts [Log.highlight(Log.color(:magenta, tsv.key_field)), Log.color(:yellow, tsv.fields * "\t")] * "\t"
+TSV.traverse tsv do |k,values,fields|
+  fvalues = values.zip(fields).collect do |v,field|
+    if String === v
+      v = v.to_f.round(2) if v =~ /^-?\d+\.\d+$/
+      v = v.to_i if v =~ /^-?\d+$/
+    end
+
+    if Numeric === v
+      if field =~ /p.?value/i
+        v = v.round(4)
+        v < 0.05 ? Log.color(:cyan, v.to_s) : v
+      else
+        v = v.round(2)
+        v < 0 ? Log.color(:red, v.to_s) : Log.color(:green, v.to_s)
+      end
+    else
+      v
+    end
+  end
+  puts [Log.color(:magenta, k), fvalues] * "\t"
+end

data/test/rbbt/test_resource.rb

@@ -17,9 +17,18 @@ module TestResource
 file 'foo' do |t|
   Open.write(t.name, "TEST")
 end
+
+rule /.*/ do |t|
+  Open.write(t.name, "bar")
+end
   EOF
 
-  claim tmp.test.work.footest, :rake, tmp.test.rakefiles.foo
+  claim tmp.test.work.footest, :rake, TestResource.tmp.test.rakefiles.foo
+
+  claim tmp.test.work.file_proc, :file_proc do |file,filename|
+    Open.write(filename, file)
+    nil
+  end
 end
 
 class TestTSV < Test::Unit::TestCase
@@ -33,11 +42,12 @@ class TestTSV < Test::Unit::TestCase
     assert TSV === TestResource.tmp.test.test_tsv.tsv
   end
 
-  def __test_rake
-    TestResource.tmp.test.work.footest.foo.read == "TEST"
-    assert TestResource.tmp.test.work.footest.foo.read == "TEST"
+  def test_rake
+    assert_equal TestResource.tmp.test.work.footest.foo.read, "TEST"
+    assert_equal TestResource.tmp.test.work.footest.bar.read, "bar"
   end
 
+
   def test_proc
     assert TestResource.tmp.test.proc.read == "PROC TEST"
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.34.4
+  version: 5.34.5
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-13 00:00:00.000000000 Z
+date: 2022-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -459,6 +459,7 @@ files:
 - share/rbbt_commands/tsv/uncollapse
 - share/rbbt_commands/tsv/unzip
 - share/rbbt_commands/tsv/values
+- share/rbbt_commands/tsv/view
 - share/rbbt_commands/tsv/write_excel
 - share/rbbt_commands/tsv/zip
 - share/rbbt_commands/watch