rbbt-util 5.21.121 → 5.21.122

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 435a3cc2ef4b1d9f0838bbd85f75e8be462cbbc3
-  data.tar.gz: 7b5549e9ceee2715acc0214e92797722b04da22a
+  metadata.gz: e3570acbd38ac9c6559cabbfc3dae257ecae6dc0
+  data.tar.gz: f863e68d23369100f01c3d818aaecad50205afff
 SHA512:
-  metadata.gz: cd02ef90d9343947194eda5f35e3dcb6edc528b431e3ecff890f12dd6f7be5e7567215fc9a01009493f1a1f106eb4e2969f95f1c8ec5b75e5153b91cfca14e69
-  data.tar.gz: 86040779d87fe38a288c7e1a274ad2cd2319cad9409dac48ed0276911b8942ee06f43ad68265138ae682aac354d1e605c0c90e3cddc1fc2ffb72a0f937c45451
+  metadata.gz: 25abb01fe95848d99d5b20e3db8f00a03921377ea291fec3c4a938e1b73deb42650c068f2f3b1349e4dfc097515176f4d0bc8d87b096e8882076e3e187985a97
+  data.tar.gz: 3a1327e9286689eadfb6c6fc6725323dcb338eb7d9bc783e2dd8d80cf1998907346c764103777fbaad582ed38471dbc76c9cc860c9ec266247292fecd041eb97

data/lib/rbbt/tsv.rb

@@ -28,7 +28,22 @@ module TSV
     end
   end
 
-  def self.setup(hash, options = {})
+  def self.str2options(str)
+    field_options, rest =  str.split("#")
+    key, fields_str = field_options.split("~")
+
+    fields = fields_str.nil? ? [] : fields_str.split(/,\s*/)
+    rest_options = rest.nil? ? {} : Misc.string2hash(rest)
+
+    {:key_field => key, :fields => fields}.merge(rest_options)
+  end
+
+  def self.setup(hash, type = nil, options = nil)
+    type, options = nil, type if options.nil? and (Hash === type or (String === type and type.include? "~"))
+    options = TSV.str2options(options) if String === options and options.include? "~"
+    options ||= {}
+    options[:type] ||= type unless type.nil?
+
     options = Misc.add_defaults options, :default_value => [], :unnamed => TSV.unnamed
     default_value = Misc.process_options options, :default_value
     hash = Misc.array2hash(hash, default_value) if Array === hash
@@ -47,7 +62,8 @@ module TSV
 
   # options shift if type.nil?
   def self.open(source, type = nil, options = nil)
-    type, options = nil, type if options.nil? and Hash === type
+    type, options = nil, type if options.nil? and (Hash === type or (String === type and type.include? "~"))
+    options = TSV.str2options(options) if String === options and options.include? "~"
     options ||= {}
     options[:type] ||= type unless type.nil?
 

data/lib/rbbt/tsv/attach.rb

@@ -215,6 +215,9 @@ module TSV
     if complete
       fill = TrueClass === complete ? nil : complete
       field_length = self.fields.length 
+      common_fields = (other.fields & self.fields)
+      other_common_pos = common_fields.collect{|f| other.fields.index f}
+      this_common_pos = common_fields.collect{|f| self.fields.index f}
       missing = other.keys - self.keys
       case type
       when :single
@@ -223,11 +226,21 @@ module TSV
         end
       when :list
         missing.each do |k|
-          self[k] = [nil] * field_length
+          values = [nil] * field_length
+          other_values = other[k]
+          other_common_pos.zip(this_common_pos).each do |o,t|
+            values[t] = other_values[o]
+          end
+          self[k] = values
         end
       when :double
         missing.each do |k|
-          self[k] = [[]] * field_length
+          values = [[]] * field_length
+          other_values = other[k]
+          other_common_pos.zip(this_common_pos).each do |o,t|
+            values[t] = other_values[o]
+          end
+          self[k] = values
         end
       when :flat
         missing.each do |k|

data/lib/rbbt/tsv/excel.rb

@@ -97,6 +97,7 @@ module TSV
       options = Misc.add_defaults options, :sep2 => /[,|]\s?/
       sheet = Misc.process_options options, :sheet
       header = Misc.process_options options, :header
+      text = Misc.process_options options, :text
 
       header = true unless header == false
       sheet ||= 0
@@ -124,7 +125,7 @@ module TSV
           end
         end
 
-        TSV.open(filename, options)
+        text ? Open.read(filename) : TSV.open(filename, options)
       end
     end
 
@@ -152,6 +153,7 @@ module TSV
       options = Misc.add_defaults options, :sep2 => /[,|]\s?/
       sheet = Misc.process_options options, :sheet
       header = Misc.process_options options, :header
+      text = Misc.process_options options, :text
 
       header = true unless header == false
       TmpFile.with_file do |filename|
@@ -178,7 +180,7 @@ module TSV
           end
         end
 
-        TSV.open(filename, options)
+        text ? Open.read(filename) : TSV.open(filename, options)
       end
     end
 

data/lib/rbbt/tsv/util.rb

@@ -334,6 +334,18 @@ module TSV
   def marshal_dump
     [info, to_hash]
   end
+
+  def to_onehot(boolean = false)
+    all_values = values.flatten.uniq.collect{|v| v.to_s}.sort
+    index = TSV.setup({}, :key_field => key_field, :fields => all_values, :type => :list)
+    index.cast = :to_i unless boolean
+    through do |key,values|
+      v = all_values.collect{|_v| values.include?(_v)}
+      v = v.collect{|_v| _v ? 1 : 0 } unless boolean
+      index[key] = v
+    end
+    index
+  end
 end
 
 class Hash

data/lib/rbbt/util/R.rb

@@ -124,7 +124,6 @@ source(interactive.script.file)
     end
   end
 
-
   def self.ruby2R(object)
     case object
     when Float::INFINITY
@@ -154,6 +153,12 @@ source(interactive.script.file)
     end
   end
 
+  def self.hash2Rargs(hash)
+    hash.collect do |k,v|
+      [k.to_s, ruby2R(v)] * "="
+    end * ", "
+  end
+
   def self.tsv(file, options = {})
     options = Misc.add_defaults :header_hash => '', :sep => / +/, :type => :list, :key_field => 'ID'
     key_field = Misc.process_options options, :key_field

data/lib/rbbt/util/R/plot.rb

@@ -1,6 +1,11 @@
 module R
   module SVG
-    def self.ggplotSVG(data, script = nil, width = nil, height = nil, options = {})
+
+    def self.ggplotPNG(*args)
+      ggplot(*args)
+    end
+
+    def self.ggplot(data, script = nil, width = nil, height = nil, options = {})
       width ||= 3
       height ||= 3
       values = []
@@ -79,7 +84,27 @@ module R
   end
 
   module PNG
-    def self.ggplotPNG(filename, data, script = nil, width = nil, height = nil, options = {})
+
+    def self.eog(data, script = nil, width = nil, height = nil, options = {})
+      TmpFile.with_file :extension => 'png' do |filename|
+        ggplot(filename, data, script, width, height, options)
+        `eog #{ filename }`
+      end
+    end
+
+    def self.eog_plot(data, script = nil, width = nil, height = nil, options = {})
+      TmpFile.with_file :extension => 'png' do |filename|
+        plot(filename, data, script, width, height, options)
+        `eog #{ filename }`
+      end
+    end
+
+
+    def self.ggplotPNG(*args)
+      ggplot(*args)
+    end
+
+    def self.ggplot(filename, data, script = nil, width = nil, height = nil, options = {})
       width ||= 3
       height ||= 3
       values = []
@@ -122,8 +147,8 @@ data = NULL
     end
 
     def self.plot(filename, data = nil, script = nil, width = nil, height = nil, options = {}, &block)
-      width ||= 200
-      height ||= 200
+      width ||= 600
+      height ||= 600
       values = []
 
       script ||= ""
@@ -210,4 +235,185 @@ data = NULL
       end
     end
   end
+
+  module GIF
+
+    def self.eog(data, frames = nil, script = nil, width = nil, height = nil, options = {})
+      TmpFile.with_file :extension => 'gif' do |filename|
+        ggplot(filename, data, frames, script, width, height, options)
+        `eog #{ filename }`
+      end
+    end
+
+    def self.eog_plot(data, frames = nil, script = nil, width = nil, height = nil, options = {})
+      TmpFile.with_file :extension => 'gif' do |filename|
+        plot(filename, data, frames, script, width, height, options)
+        `eog #{ filename }`
+      end
+    end
+
+
+    def self.ggplot(filename, data, frames = nil, script = nil, width = nil, height = nil, options = {})
+      width ||= 3
+      height ||= 3
+      values = []
+
+      sources = [:plot, options[:source]].flatten.compact
+
+      data.each do |k,v|
+        v = Array === v ? v : [v]
+        next if v == "NA" or v.nil? or v.include? "NA" or v.include? nil
+        values = v
+        break
+      end
+      values = [values] unless Array === values
+      field_classes = values.collect do |v| 
+        case v
+        when FalseClass, TrueClass
+          "'logical'"
+        when Numeric
+          "'numeric'"
+        when String
+          if v.strip =~ /^[-+]?[\d\.]+$/
+            "'numeric'"
+          else
+            "'character'"
+          end
+        when Symbol
+          "'factor'"
+        else
+          ":NA"
+        end
+      end
+      options[:R_open] ||= "colClasses=c('character'," + field_classes * ", " + ')'
+
+      delay = options[:delay]
+      delay = 10 if delay.nil?
+
+      frames = data.keys if frames.nil?
+      frames = (1..frames).to_a if Integer === frames
+
+      data.R <<-EOF, :plot, options
+frames = #{R.ruby2R frames}
+for (frame in seq(1, length(frames))){
+  frame.value = frames[frame]
+  frame.str = sprintf("%06d", frame)
+  plot = { #{script} }
+
+  ggsave(paste('#{filename}', frame.str, 'tmp.png', sep="."), plot, width = #{R.ruby2R width}, height = #{R.ruby2R height})
+}
+data = NULL
+      EOF
+
+      CMD.cmd("convert #{filename}.*.tmp.png -set delay #{delay} -loop 0 #{filename} && rm #{filename}.*.tmp.png")
+    end
+
+    def self.plot(filename, data = nil, frames = nil, script = nil, width = nil, height = nil, options = {}, &block)
+      width ||= 600
+      height ||= 600
+      values = []
+
+      script ||= ""
+      if block_given?
+        s = StringIO.new
+        class << s
+          def method_missing(name, *args)
+            name = name.to_s
+            if name[-1] == '='
+              arg = args.first
+              value = if String === arg
+                        arg
+                      else
+                        R.ruby2R arg
+                      end
+              add("" << name[0..-2] << "=" << value)
+            else
+              args_strs = []
+              args.each do |arg|
+                value = if String === arg
+                          arg
+                        else
+                          R.ruby2R arg
+                        end
+                args_strs << value
+              end
+              add("" << name << "(" << args_strs * ", " << ")")
+            end
+          end
+
+          def add(line)
+            self.write line << "\n"
+          end
+        end
+        block.call(s)
+        s.rewind
+        script << "\n" << s.read
+      end
+      sources = [:plot, options[:source]].flatten.compact
+      
+      delay = options[:delay]
+      delay = 10 if delay.nil?
+
+      frames = data.keys if frames.nil?
+      frames = (1..frames).to_a if Integer === frames
+
+      if data
+        data.each do |k,v|
+          v = Array === v ? v : [v]
+          next if v == "NA" or v.nil? or v.include? "NA" or v.include? nil
+          values = v
+          break
+        end 
+
+        values = [values] unless values.nil? or Array === values
+
+        field_classes = values.collect do |v| 
+          case v
+          when FalseClass, TrueClass
+            "'logical'"
+          when Numeric
+            "'numeric'"
+          when String
+            if v.strip =~ /^[-+]?[\d\.]+$/
+              "'numeric'"
+            else
+              "'character'"
+            end
+          when Symbol
+            "'factor'"
+          else
+            ":NA"
+          end
+        end
+
+        options[:R_open] ||= "colClasses=c('character'," + field_classes * ", " + ')' if field_classes.any?
+
+        data.R <<-EOF, :plot, options
+frames = #{R.ruby2R frames}
+for (frame in seq(1, length(frames))){
+  frame.value = frames[frame]
+  frame.str = sprintf("%06d", frame)
+  plot = { #{script} }
+
+  png(paste('#{filename}', frame.str, 'tmp.png', sep="."), #{ width }, #{ height })
+  { #{script} }
+  dev.off()
+}
+data = NULL
+        EOF
+      else
+        R.run <<-EOF, :plot, options
+frames = #{R.ruby2R frames}
+for (frame in seq(1, length(frames))){
+  frame.value = frames[frame]
+  frame.str = sprintf("%06d", frame)
+  png(paste('#{filename}', frame.str, 'tmp.png', sep="."), #{ width }, #{ height })
+  { #{script} }
+  dev.off()
+}
+        EOF
+      end
+      CMD.cmd("convert #{filename}.*.tmp.png -set delay #{delay} -loop 0 #{filename} && rm #{filename}.*.tmp.png")
+    end
+  end
 end

data/lib/rbbt/util/tmpfile.rb

@@ -28,6 +28,9 @@ module TmpFile
   end
 
   def self.with_file(content = nil, erase = true, options = {})
+    options, content, erase = content, nil, true if Hash === content
+    options, erase = erase, true if Hash === erase 
+
     prefix = options[:prefix] || ""
     tmpfile = tmp_file prefix
     if options[:extension]

data/share/Rlib/util.R

@@ -737,10 +737,20 @@ rbbt.psd <- function(m){
     return(p)
 }
 
+rbbt.impute <- function(data){
+    m = as.matrix(data)
+    if (sum(is.na(m)) == 0){
+        return (m)
+    }else{
+        rbbt.require('Hmisc')
+        m.i = impute(m)
+        return(m.i)
+    }
+}
+
 rbbt.fix_distance <- function(data){
-    rbbt.require('Hmisc')
 
-    m = impute(as.matrix(data))
+    m = rbbt.impute(data)
 
     p <- rbbt.psd(m)
     p <- m

data/test/rbbt/test_tsv.rb

@@ -387,7 +387,7 @@ row2    A    B    Id3
     end
   end
 
-  def test_flat_merge
+  def test_flat_merge2
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
 row1    a|aa|aaa    b    Id1|Id2
@@ -408,12 +408,39 @@ row1   a   aa   aaa
 row2   b  bbb bbbb bb
     EOF
 
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :merge => false, :type => :flat)
+      assert_equal ["a", "aa", "aaa"], tsv["row1"]
+    end
+  end
+
+  def test_flat_with_field
+    content =<<-EOF
+#Id    ValueA
+row1   a   aa   aaa
+row2   b  bbb bbbb bb
+    EOF
+
     TmpFile.with_file(content) do |filename|
       tsv = TSV.open(filename, :sep => /\s+/, :merge => false, :type => :flat, :fields => ["ValueA"])
       assert_equal ["a", "aa", "aaa"], tsv["row1"]
     end
   end
 
+  def test_flat_with_field_header
+    content =<<-EOF
+#: :type=:flat
+#Id    ValueA
+row1   a   aa   aaa
+row2   b  bbb bbbb bb
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :sep => /\s+/, :merge => false, :fields => ["ValueA"])
+      assert_equal ["a", "aa", "aaa"], tsv["row1"]
+    end
+  end
+
   def test_flat_key
     content =<<-EOF
 #Id    ValueA 
@@ -525,10 +552,8 @@ row2    A    B    Id3
 
     TmpFile.with_file(content) do |filename|
       tsv = TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "ValueA", :fields => ["OtherID"], :merge => true)
-      ppp tsv
       assert tsv["aaa"].include? "Id1"
       assert tsv["aaa"].include? "Id2"
- 
     end
   end
 
@@ -554,10 +579,6 @@ row2    A AA AAA
     assert_equal 10000, tsv.keys.length + 2
   end
 
-  def test_merge_key_field
-    assert TSV.open('/home/mvazquezg/git/workflows/genomics/share/gene_ages', :key_field => "FamilyAge", :merge => true,  :persist => false, :zipped => true, :type => :double).values.first.length > 1
-  end
-
   def test_flat_merge
     content =<<-EOF
 #Id    ValueA    ValueB    OtherID
@@ -600,4 +621,21 @@ row2   b  bbb bbbb bb
       assert tsv.include? 'aa'
     end
   end
+
+
+  def test_string_options
+    content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A|aAa    B    Id3
+row3    AA    BB|BBB    Id3|Id2
+    EOF
+
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.open(filename, :flat, 'Id~OtherID#:sep=/\s+/')
+      assert_equal %w(Id1 Id2), tsv["row1"]
+ 
+    end
+  end
 end
+

data/test/rbbt/util/R/test_plot.rb

@@ -0,0 +1,40 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
+require 'rbbt/util/R'
+require 'rbbt/util/R/plot'
+
+class TestRPlot < Test::Unit::TestCase
+  def __test_ggplotgif
+
+    Log.severity = 0
+    x = R.eval_a 'rnorm(100, 0, 1)'
+    data = TSV.setup(x, "Num~#:type=:single#:cast=:to_f")
+
+    data.add_field "Val" do |k,v|
+      k
+    end
+
+    R::GIF.eog(data, (0..10).to_a.collect{|i| 100.0 / 10**i}, <<-EOF, nil, nil, :delay => 100)
+rbbt.require('reshape')
+ggplot(data) + geom_density(bw=frame.value, aes(x=Val))
+    EOF
+
+  end
+
+  def test_gif
+
+    Log.severity = 0
+    x = R.eval_a 'rnorm(100, 0, 1)'
+    data = TSV.setup(x, "Num~#:type=:single#:cast=:to_f")
+
+    data.add_field "Val" do |k,v|
+      k
+    end
+
+    R::GIF.eog_plot(data, (0..10).to_a.collect{|i| 100.0 / 10**i}, <<-EOF, 400, 400, :delay => 10)
+rbbt.require('reshape')
+plot(density(data$Val, bw=frame.value))
+    EOF
+
+  end
+end
+

data/test/rbbt/util/test_R.rb

@@ -26,5 +26,11 @@ data = data + 1
       EOF
     end
   end
+
+  def test_hash2Rargs
+    hash = {:verbose => false, :iter => 1000, :method => 'kmeans'}
+    str = R.hash2Rargs(hash)
+    assert_equal "verbose=FALSE, iter=1000, method='kmeans'", str
+  end
 end
 

data/test/rbbt/util/test_cmd.rb

@@ -5,27 +5,27 @@ require 'test/unit'
 class TestCmd < Test::Unit::TestCase
 
   def test_cmd_option_string
-    assert_equal("--user-agent firefox", CMD.process_cmd_options("--user-agent" => "firefox"))
-    assert_equal("--user-agent=firefox", CMD.process_cmd_options("--user-agent=" => "firefox"))
+    assert_equal("--user-agent 'firefox'", CMD.process_cmd_options("--user-agent" => "firefox"))
+    assert_equal("--user-agent='firefox'", CMD.process_cmd_options("--user-agent=" => "firefox"))
     assert_equal("-q", CMD.process_cmd_options("-q" => true))
     assert_equal("", CMD.process_cmd_options("-q" => nil))
     assert_equal("", CMD.process_cmd_options("-q" => false))
 
-    assert(CMD.process_cmd_options("--user-agent" => "firefox", "-q" => true) =~ /--user-agent firefox/)
+    assert(CMD.process_cmd_options("--user-agent" => "firefox", "-q" => true) =~ /--user-agent 'firefox'/)
     assert(CMD.process_cmd_options("--user-agent" => "firefox", "-q" => true) =~ /-q/)
   end
 
   def test_cmd
     assert_equal("test\n", CMD.cmd("echo '{opt}' test").read)
     assert_equal("test", CMD.cmd("echo '{opt}' test", "-n" => true).read)
-    assert_equal("test2\n", CMD.cmd("cut", "-f" => 2, "-d" => '" "', :in => "test1 test2").read)
+    assert_equal("test2\n", CMD.cmd("cut", "-f" => 2, "-d" => ' ', :in => "test1 test2").read)
   end
 
   def test_pipe
     assert_equal("test\n", CMD.cmd("echo test", :pipe => true).read)
     assert_equal("test\n", CMD.cmd("echo '{opt}' test", :pipe => true).read)
     assert_equal("test", CMD.cmd("echo '{opt}' test", "-n" => true, :pipe => true).read)
-    assert_equal("test2\n", CMD.cmd("cut", "-f" => 2, "-d" => '" "', :in => "test1 test2", :pipe => true).read)
+    assert_equal("test2\n", CMD.cmd("cut", "-f" => 2, "-d" => ' ', :in => "test1 test2", :pipe => true).read)
   end
 
   def test_error

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.21.121
+  version: 5.21.122
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-03 00:00:00.000000000 Z
+date: 2018-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -452,6 +452,7 @@ files:
 - test/rbbt/tsv/test_util.rb
 - test/rbbt/util/R/test_eval.rb
 - test/rbbt/util/R/test_model.rb
+- test/rbbt/util/R/test_plot.rb
 - test/rbbt/util/concurrency/processes/test_socket.rb
 - test/rbbt/util/concurrency/test_processes.rb
 - test/rbbt/util/concurrency/test_threads.rb
@@ -522,6 +523,7 @@ test_files:
 - test/rbbt/util/misc/test_bgzf.rb
 - test/rbbt/util/test_concurrency.rb
 - test/rbbt/util/test_cmd.rb
+- test/rbbt/util/R/test_plot.rb
 - test/rbbt/util/R/test_eval.rb
 - test/rbbt/util/R/test_model.rb
 - test/rbbt/util/test_log.rb