rbbt-util 5.14.32 → 5.14.33

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6cb343242db38c910d25a554cd9e4bec1efbe8b1
-  data.tar.gz: 1da7fa8d52a241541bc8266145826acd133004c0
+  metadata.gz: e156f608c54a1a71f1d7892428698d857075c4e7
+  data.tar.gz: 535f2b8b01c561226a389050cbb90b31b82ac9e2
 SHA512:
-  metadata.gz: 18a7224b05d0c8cf4b73c3a8f08c2dc65cc3894e63225e8e5348e67ec8c26fbe6749277f3a18197b64a514744e0f70c5b76141bfce501d4c12faf0f3d1292174
-  data.tar.gz: 6718fb810d5e471361e93fb1563b190627e09a7be17e95b5f6b5ec6722fddc025a09d466ecbc1e5d64bd928ca47fee8f6ca4b23ec1456b3cfd09489ea9dbb90b
+  metadata.gz: 8bad7add5780e3764172ffb11c87b8b3cebc5267fe2f11dbe0efcdd46cebd92019b669346d31ff08a0ab32321356c2fc21ef5f776d1ce188d8527f2b1a4044c9
+  data.tar.gz: 543e58c6022ccacbbd164aa5a97df5ab7ab9c168582ec212b5b5cfce7714b02ca1f5b74b2c59a50c27297aa9113ccbf548ce62e3aafa3d530371361acb180bc3

data/lib/rbbt/association.rb

@@ -288,10 +288,15 @@ module Association
   def self.index(file, options = {}, persist_options = nil)
     options = {} if options.nil?
     options = Misc.add_defaults options, :persist => true, :undirected => false
-    persist_options = Misc.pull_keys options, persist_options if persist_options.nil?
+    persist_options = Misc.pull_keys options, :persist if persist_options.nil?
 
-    Persist.persist_tsv(file, nil, options, {:persist => true, :prefix => "Association Index"}.merge(persist_options).merge(:engine => TokyoCabinet::BDB, :serializer => :clean)) do |assocs|
+    expanded_persist_options = {:persist => true, :prefix => "Association Index"}.
+      merge(persist_options).
+      merge(:engine => TokyoCabinet::BDB, :serializer => :clean)
+
+    Persist.persist_tsv(file, nil, options, expanded_persist_options) do |assocs|
       undirected = options[:undirected]
+      recycle = options[:recycle]
       if file
         tsv = TSV === file ? file : Association.open(file, options, persist_options.merge(:persist => false))
 
@@ -330,8 +335,18 @@ module Association
                 next if values.empty?
                 next if source.nil? or source.empty?
                 next if values.empty?
+
                 targets = values.first
-                rest = Misc.zip_fields values[1..-1]
+
+                rest =  values[1..-1]
+
+                size = values.first ? values.first.length : 0
+
+                rest.each_with_index do |list,i|
+                  list.replace [list.first] * size if list.length == 1
+                end if recycle and size > 1
+
+                rest = Misc.zip_fields rest
 
                 annotations = rest.length > 1 ?
                   targets.zip(rest) :

data/lib/rbbt/tsv/attach/util.rb

@@ -23,7 +23,7 @@ module TSV
             when other.type == :single
               new_values = [other[key]]
             else
-              other_values = other[key]
+              other_values = other[key] || [nil] * other.fields.length
               new_values = field_positions.collect do |pos|
                 pos == :key ? key : other_values[pos]
               end

data/lib/rbbt/tsv/matrix.rb

@@ -1,6 +1,14 @@
+require 'rbbt/association'
+
 module TSV
-  def self.read_matrix(tsv, field_format = "ID", value_format = "Value")
+  def self.read_matrix(tsv, field_format = "ID", value_format = "Value", *others)
     tsv = TSV.open(tsv) unless TSV === tsv
+    
+
+    if others.any?
+      other_tsv = tsv.slice(others)
+      tsv = tsv.slice(tsv.fields - others)
+    end
 
     key_field, *fields = tsv.all_fields
     options = tsv.options.merge(:key_field => key_field, :fields => [field_format, value_format], :type => :double, :cast => nil)
@@ -15,6 +23,26 @@ module TSV
       [key, [fields, values]]
     end
 
-    dumper.stream
+    res = TSV.open(dumper.stream, options)
+    if others.any?
+      other_tsv = other_tsv.to_double
+      res.attach other_tsv, :one2one => true
+    else
+      res
+    end
+  end
+
+  def matrix_melt(*args)
+    tsv = TSV.read_matrix(self, *args)
+
+    melt = Association.index tsv, :persist => false, :recycle => true
+    source_field,_sep,target_field = melt.key_field.partition "~"
+    melt.add_field source_field do |k,v|
+      k.partition("~").first
+    end
+    melt.add_field target_field do |k,v|
+      k.partition("~").last
+    end
+    melt
   end
 end

data/lib/rbbt/tsv/util.rb

@@ -69,8 +69,12 @@ module TSV
     case
     when (defined? Step and Step === file)
       file.path
-    when String === file
+    when Path === file
       file
+    when (String === file and (Open.exists? file or Open.remote? file))
+      file
+    when String === file 
+      "String-#{Misc.digest file}"
     when file.respond_to?(:filename)
       file.filename
     when file.respond_to?(:gets)

data/lib/rbbt/util/R/eval.rb

@@ -12,41 +12,90 @@ module Rserve
 end
 
 module R
-  PID = Process.pid
+  SESSION = ENV["RServe-session"] || "Session-PID-" + Process.pid.to_s
+
+  def self.socket_file
+    @@socket_file ||= Rbbt.tmp.R_sockets[R::SESSION].find
+  end
+
+  def self.lockfile
+    @@lockfile ||= socket_file + '.lock'
+  end
+
+  def self.workdir
+    @@workdir ||= socket_file + '.wd'
+  end
+
+  def self.pid_file
+    @@pidfile ||= File.join(workdir, 'pid')
+  end
+
+  def self.clear
+    Log.warn "Clearing Rserver session #{SESSION}, PID #{@@instance_process}"
+    @@instance = nil
+    if defined? @@instance_process and @@instance_process and Misc.pid_exists? @@instance_process
+      begin
+        Process.kill :INT, @@instance_process
+      rescue Exception
+      end
+    end
+    FileUtils.rm_rf pid_file if File.exists? pid_file
+    FileUtils.rm_rf socket_file if File.exists? socket_file
+    FileUtils.rm_rf lockfile if File.exists? lockfile
+    FileUtils.rm_rf workdir if File.exists? workdir
+  end
+
   def self.instance
     @@instance ||= begin
-                     @@socket_file = Rbbt.tmp.R_sockets[R::PID].find
 
-                     FileUtils.mkdir_p File.dirname(@@socket_file) unless File.directory?(File.dirname(@@socket_file))
+                     clear if File.exists? pid_file and ! Misc.pid_exists?(Open.read(pid_file).strip.to_i)
+
+                     FileUtils.mkdir_p File.dirname(socket_file) unless File.directory?(File.dirname(socket_file))
+                     FileUtils.mkdir_p workdir unless File.directory? workdir
+
+                     at_exit do
+                       self.clear
+                     end unless defined? @@instance_process
 
                      begin
 
-                       if not File.exists? @@socket_file
+                       if not File.exists? socket_file
 
-                         @@instance_process = Process.fork do
-                           args = %w(CMD Rserve --vanilla --quiet --RS-socket)
-                           args << "'#{@@socket_file}'"
+                         sh_pid = Process.fork do
+                           #args = %w(CMD Rserve --vanilla --quiet --RS-socket)
+                           args = %w(--vanilla --quiet --RS-socket)
+                           args << "'#{socket_file}'"
+                           args << "--RS-workdir"
+                           args << "'#{workdir}'"
+                           args << "--RS-pidfile"
+                           args << "'#{pid_file}'"
 
-                           bin_path = "R"
+                           bin_path = File.join(ENV["R_HOME"], "bin/Rserve")
                            cmd = bin_path + " " + args*" "
                            exec(ENV, cmd)
                          end
-                         sleep 1
+                         while not File.exists? pid_file
+                           sleep 0.5
+                         end
+                         @@instance_process = Open.read(pid_file).to_i
+                         Log.info "New Rserver session stated with PID (#{sh_pid}) #{@@instance_process}: #{SESSION}"
                        end
 
+                       i = Rserve::Connection.new :hostname => socket_file
+
                        begin
-                        i = Rserve::Connection.new :hostname => @@socket_file
+                        FileUtils.mkdir workdir unless File.exists? workdir
+                        i.eval "setwd('#{workdir}');"
                         i.eval "source('#{UTIL}');" 
-                        wdir = @@socket_file + '.wd'
-                        FileUtils.mkdir wdir unless File.exists? wdir
-                        i.eval "setwd('#{wdir}');"
                         i
                        rescue Exception
+                         Log.exception $!
                          raise TryAgain
                        end
                      rescue Exception
+                       Log.exception $!
                        Process.kill :INT, @@instance_process if defined? @@instance_process and @@instance_process
-                       FileUtils.rm @@socket_file if File.exists? @@socket_file
+                       FileUtils.rm socket_file if File.exists? socket_file
                        retry if TryAgain === $!
                        raise $!
                      end
@@ -54,7 +103,20 @@ module R
   end
 
   def self._eval(cmd)
-    instance.eval(cmd)
+    Misc.lock lockfile do 
+      times = 2
+      begin
+        instance.eval(cmd)
+      rescue Rserve::Connection::EvalError
+        times = times - 1
+        if times > 0
+          clear
+          retry 
+        else
+          raise $!
+        end
+      end
+    end
   end
 
   def self.eval_a(cmd)

data/lib/rbbt/util/R/model.rb

@@ -16,11 +16,25 @@ module R
 
   self.model_dir = Rbbt.var.R.models
   class Model
+    R_METHOD = :debug
 
     attr_accessor :name, :formula
-    def initialize(name, formula)
+    def initialize(name, formula, options = {})
       @name = name
       @formula = formula
+      @options = options || {}
+    end
+
+    def colClasses(tsv)
+      "c('character', " << 
+      (tsv.fields.collect{|f| R.ruby2R(@options[f] ? @options[f].to_s : ":NA") } * ", ") <<
+      ")"
+    end
+
+    def r_options(tsv)
+      {:R_open => "colClasses=#{colClasses(tsv)}", 
+        :R_method => (@options[:R_method] || R_METHOD), 
+          :source => @options[:source]}
     end
 
     def model_file
@@ -28,10 +42,11 @@ module R
     end
 
     def update(tsv, field = "Prediction")
-      tsv.R <<-EOF, :R_method => :eval
+      tsv.R <<-EOF, r_options(tsv)
 model = rbbt.model.load('#{model_file}');
 model = update(model, data);
 save(model, file='#{model_file}');
+data = NULL
       EOF
     end
 
@@ -49,9 +64,10 @@ save(model, file='#{model_file}');
 
     def predict(tsv, field = "Prediction")
       tsv = Model.groom tsv, formula 
-      tsv.R <<-EOF, :R_method => :eval, :key => tsv.key_field
+      tsv.R <<-EOF, r_options(tsv)
 model = rbbt.model.load('#{model_file}');
-data$#{field} = predict(model, data);
+data.groomed = rbbt.model.groom(data,formula=#{formula})
+data$#{field} = predict(model, data.groomed);
       EOF
     end
 
@@ -67,9 +83,10 @@ data$#{field} = predict(model, data);
       tsv = Model.groom(tsv, formula)
 
       FileUtils.mkdir_p File.dirname(model_file) unless File.exists?(File.dirname(model_file))
-      tsv.R <<-EOF, :R_method => :shell
+      tsv.R <<-EOF, r_options(tsv)
 model = rbbt.model.fit(data, #{formula}, method=#{method}#{args_str})
 save(model, file='#{model_file}')
+data = NULL
       EOF
     end
   end

data/lib/rbbt/util/R.rb

@@ -93,7 +93,7 @@ source('#{UTIL}');
     when Symbol
       "#{ object }"
     when String
-      "'#{ object }'"
+      object[0] == ":" ? object[1..-1] : "'#{ object }'"
     when Fixnum, Float
       object
     when TrueClass
@@ -120,6 +120,7 @@ module TSV
   def R(script, source = nil, open_options = {})
     open_options, source = source, nil if Hash === source
 
+    source ||= Misc.process_options open_options, :source
     source = [source] if String === source
 
     require_sources  = source.collect{|source|
@@ -153,6 +154,7 @@ data = rbbt.tsv('#{f}'#{tsv_R_option_str});
 
 ## Resaving data
 if (! is.null(data)){ rbbt.tsv.write('#{f}', data); }
+NULL
       EOF
 
 

data/lib/rbbt/workflow/step/run.rb

@@ -319,8 +319,8 @@ class Step
         set_info :pid, nil
       ensure
         RbbtSemaphore.post_semaphore(semaphore) if semaphore
+        Kernel.exit! 0
       end
-      Kernel.exit! 0
     end
     Process.detach(@pid)
     self

data/share/Rlib/util.R

@@ -228,6 +228,29 @@ rbbt.parse <- function(filename){
     return(parse(text=paste(lines[from:to],sep="\n")));
 }
 
+rbbt.pull.keys <- function(items, key){
+    pulled = list()
+    rest = list()
+
+    names = names(items)
+
+    prefix = paste("^",key,'.',sep='')
+    matches = grep(prefix, names)
+
+    for (i in seq(1,length(names))){
+        if (i %in% matches){
+            name = names[i]
+            name = sub(prefix, "", name)
+            pulled[[name]] = items[[i]]
+        }else{
+            name = names[i]
+            rest[[name]] = items[[i]]
+        }
+    }
+    
+    list(pulled=pulled, rest=rest)
+}
+
 rbbt.run <- function(filename){
     rbbt.reload();
     eval(rbbt.parse(filename), envir=globalenv());
@@ -314,6 +337,14 @@ rbbt.ddd <- function(o){
     cat("\n", file = stderr())
 }
 
+# From: http://ryouready.wordpress.com/2008/12/18/generate-random-string-name/
+rbbt.random_string <- function(n=1, length=12){
+    randomString <- c(1:n)
+    for (i in 1:n){
+        randomString[i] <- paste(sample(c(0:9, letters, LETTERS), length, replace=TRUE), collapse="")
+    }
+    return(randomString)
+}
 
 
 # {{{ MODELS
@@ -332,7 +363,7 @@ rbbt.model.groom <- function(data, variables = NULL, classes = NULL, formula = N
         variables = names[names %in% all.vars(formula)]
     }
 
-    data.groomed = data[,variables]
+    data.groomed = data[,variables,drop=F]
 
     if (! is.null(classes)){
         if (is.character(classes)){ classes = rep(classes,dim(data.groomed)[2]) }
@@ -362,29 +393,6 @@ rbbt.model.load <- function(file, force = F){
     rbbt.loaded.models[[file]]
 }
 
-rbbt.pull.keys <- function(items, key){
-    pulled = list()
-    rest = list()
-
-    names = names(items)
-
-    prefix = paste("^",key,'.',sep='')
-    matches = grep(prefix, names)
-
-    for (i in seq(1,length(names))){
-        if (i %in% matches){
-            name = names[i]
-            name = sub(prefix, "", name)
-            pulled[[name]] = items[[i]]
-        }else{
-            name = names[i]
-            rest[[name]] = items[[i]]
-        }
-    }
-    
-    list(pulled=pulled, rest=rest)
-}
-
 rbbt.model.add_fit <- function(data, formula, method, classes=NULL, ...){
     data.groomed = rbbt.model.groom(data, formula=formula, classes = classes);
 

data/share/rbbt_commands/app/start

@@ -3,11 +3,14 @@
 require 'rbbt-util'
 require 'rbbt/util/simpleopt'
 
-options = SOPT.get "-e--environment*:-p--port*:-s--server*:-f--finder"
+options = SOPT.get "-e--environment*:-p--port*:-s--server*:-f--finder-R--Rserve_session*"
 options[:Port] ||= options[:port]
 
+
 app = ARGV.shift
 
+ENV["RServe-session"] = options[:RServe_session] || app
+
 app_dir = Rbbt.etc.app_dir.exists? ? Path.setup(Rbbt.etc.app_dir.read.strip) : Rbbt.apps.find
 
 app_dir = app_dir[app]

data/share/rbbt_commands/workflow/server

@@ -13,10 +13,14 @@ require 'rbbt/rest/file_server'
 require 'rbbt/rest/knowledge_base'
 require 'rbbt/rest/helpers'
 
-options = SOPT.get "-e--environment*:-p--port*:-s--server*:-b--bind*:-e--environment*"
+options = SOPT.get "-e--environment*:-p--port*:-s--server*:-b--bind*:-e--environment*:-R--RServe_session*"
 
 workflow = ARGV.first
 
+ENV["RServe-session"] = options[:RServe_session] || workflow
+
+raise rbbt_usage unless workflow
+
 wf = Workflow.require_workflow workflow
 
 $title = wf.to_s

data/test/rbbt/tsv/test_matrix.rb

@@ -0,0 +1,20 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
+require 'rbbt/tsv'
+require 'rbbt/tsv/matrix'
+
+class TestClass < Test::Unit::TestCase
+  def test_matrix
+    tsv = TSV.open <<-EOF, :sep => " ", :type => :list
+#X Y Z Type Case
+a 2 3 v l
+A 4 7 v L
+m 2 3 c l
+n 4 7 c L
+    EOF
+    tsv = TSV.open(tsv.to_s)
+
+    melt =  tsv.matrix_melt("Letter", "Value", "Type", "Case")
+    assert_equal "L", melt["A~Y"][2]
+  end
+end
+

data/test/rbbt/util/R/test_model.rb

@@ -25,7 +25,7 @@ class TestRModel < Test::Unit::TestCase
     assert pred > y and pred < y + 4
   end
 
-  def test_add_fit
+  def _test_add_fit
     tsv = TSV.open datafile_test('dose_response'), :type => :list
     tsv = tsv.slice(["Dose", "Response"])
 
@@ -49,4 +49,41 @@ data = rbbt.model.inpute(data, CI ~ Dose, method=drm, classes='numeric', fct=LL.
 
     assert_equal result.size, result.column("CI").values.flatten.reject{|p| p.nil? or p.empty? or p == "NA"}.length
   end
+
+  def test_ab_surv_corr
+    require 'rbbt/workflow'
+    Workflow.require_workflow "Miller"
+
+    require 'rbbt/util/R/model'
+    require 'rbbt/util/R/svg'
+
+    antibody = "14-3-3-Zeta(C)_GBL9006927"
+
+    rppa = Miller.RPPA.data.tsv
+    rppa.attach Miller.RPPA.labels
+    viability = Miller.Viability.data.tsv
+
+    viability.add_field "Perturbation" do |compound,values|
+      values["Dose"].collect do |dose|
+        compound.split("-").flatten.zip(dose.split("-")).collect{|p| p * "="} * "-"
+      end
+    end
+
+    viability = viability.reorder "Perturbation", nil, :zipped => true
+    compounds = viability.column("Compound").flatten.uniq
+
+    rppa = rppa.select("Compound"){|c| ! c.include? "-" and compounds.include? c}
+    rppa = rppa.slice([antibody,"Compound", "Dose"])
+    rppa.rename_field antibody, "RPPA"
+
+    model = R::Model.new "viability", "Effect ~ Dose * Compound", "Compound" => :factor
+
+    model.fit(viability.select("Compound"){|c| ! c.include? "-"}, 'drm', :fct => ":LL.4()")
+
+    rppa = model.predict(rppa, "Prediction")
+
+    plot_script = "plot<-ggplot(data=data) + geom_point(aes(x=RPPA, y=Prediction, color=Compound));"
+
+    puts R::SVG.ggplotSVG rppa, plot_script, 7, 7, :R_method => :eval 
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-util
 version: !ruby/object:Gem::Version
-  version: 5.14.32
+  version: 5.14.33
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-02 00:00:00.000000000 Z
+date: 2014-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -336,6 +336,7 @@ files:
 - test/rbbt/tsv/test_filter.rb
 - test/rbbt/tsv/test_index.rb
 - test/rbbt/tsv/test_manipulate.rb
+- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_parallel.rb
 - test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_stream.rb
@@ -448,6 +449,7 @@ test_files:
 - test/rbbt/tsv/test_field_index.rb
 - test/rbbt/tsv/test_parallel.rb
 - test/rbbt/tsv/test_index.rb
+- test/rbbt/tsv/test_matrix.rb
 - test/rbbt/tsv/test_change_id.rb
 - test/rbbt/tsv/test_parser.rb
 - test/rbbt/tsv/test_stream.rb