rbbt-dm 1.1.58 → 1.1.60

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 286385d90b276d30cd5e1b21ae38c5e6a203e2ce3ac10673c434c19a2f45cfb1
-  data.tar.gz: 7879d74a364886ea8cb507be51c4979cfb598bdb273f948c3c3930a5dce199e6
+  metadata.gz: c1f04f874e4f4d9e6b7b4dd19fb7f8e1c5c0926f6621cb246324294dab0c9664
+  data.tar.gz: ddc35ddc2c747a98405dd021e4268379f667859d3f62d0bf4785457dc8ab952c
 SHA512:
-  metadata.gz: b82c77bd736c8422e49c4dc83b63d6a91da6e76857af4b5cf5aff0a9a58b4147bc50b49b1b1534e8b07ca1bce5f6a5a673c5d688fb8cd7856623370d19fd1bda
-  data.tar.gz: 1b267a85ab600b878e99f414f725255cf086165a27f8cdec42ed83349b4f36bdb9e29615e0aaada9b30f098df8382e4778cebaf2b8649e17b8985e79d9b8bd23
+  metadata.gz: cac8f02f1d4a34658f8cf3fb8f226964b83ae56ebb79fb854f7485b6deb0364a80a7620e1f1b55113e9c18e2b48c43ec080de0eb046a3e1fb762896970a2332e
+  data.tar.gz: b72acf9908a04cbd0812772456cfe23f52385b830405231a520642a6e08b96773c01f8bb112758ca775e5a86d8ab923d1c325b56efe6561611d7102af7fcfcdf

data/lib/rbbt/matrix/barcode.rb

@@ -9,7 +9,7 @@ source('#{Rbbt.share.R['barcode.R'].find}')
 rbbt.GE.barcode.mode(#{ R.ruby2R self.data_file }, #{ R.ruby2R outfile }, #{ R.ruby2R factor })
     EOF
 
-    R.run(cmd)
+    R.run(cmd, :monitor => true)
   end
 
   def barcode_ruby(outfile, factor = 2)
@@ -55,7 +55,7 @@ source('#{Rbbt.share.R['barcode.R'].find}')
 rbbt.GE.activity_cluster(#{ R.ruby2R self.data_file }, #{ R.ruby2R outfile }, #{R.ruby2R key_field}, #{R.ruby2R clusters})
     EOF
 
-    R.run(cmd)
+    R.run(cmd, :monitor => true)
   end
 
 

data/lib/rbbt/matrix/differential.rb

@@ -1,4 +1,5 @@
 require 'rbbt/util/R'
+require 'rbbt/matrix'
 
 class RbbtMatrix
   def differential(main, contrast, path = nil)
@@ -31,9 +32,7 @@ class RbbtMatrix
         trend = false
         two_channel = false
       when 'fpkm'
-        log2 = true
-        trend = true
-        two_channel = false
+        type = "DESeq"
       when 'log2 ratio', 'transformed count'
         log2 = false
         trend = false
@@ -53,6 +52,7 @@ class RbbtMatrix
 source('#{Rbbt.share.R["MA.R"].find(:lib)}')
 
 data = rbbt.dm.matrix.differential(#{ R.ruby2R data_file }, 
+  type = #{R.ruby2R type},
   main = #{R.ruby2R(main_samples)}, 
   contrast = #{R.ruby2R(contrast_samples)}, 
   log2=#{ R.ruby2R log2 }, 

data/lib/rbbt/matrix.rb

@@ -140,6 +140,21 @@ class RbbtMatrix
     [main_samples, contrast_samples]
   end
 
+  def transpose(id = nil)
+    name = data_file =~ /:>/ ? File.basename(data_file) : data_file
+
+    file = Persist.persist(data_file, :tsv, :prefix => "Transpose", :check => [data_file],  :dir => RbbtMatrix.matrix_dir.values, :no_load => true) do
+
+      data = data_file.tsv(:cast => :to_f, :type => :double).transpose(id)
+
+      data.to_list{|v| v.length > 1 ? Misc.mean(v) : v }
+    end
+    subsets = self.subsets
+    matrix = RbbtMatrix.new file, labels, value_type, key_field, organism
+    matrix.subsets = subsets
+    matrix
+  end
+
   def to_average(identifiers = nil)
     name = data_file =~ /:>/ ? File.basename(data_file) : data_file
 

data/lib/rbbt/network/paths.rb

@@ -3,8 +3,16 @@ require 'fc'
 module Paths
 
   def self.dijkstra(adjacency, start_node, end_node = nil, max_steps = nil)
+
     return nil unless adjacency.include? start_node
 
+    case end_node
+    when String
+      return nil unless adjacency.values.flatten.include? end_node
+    when Array
+      return nil unless (adjacency.values.flatten & end_node).any?
+    end
+
     active = FastContainers::PriorityQueue.new(:min)
     distances = Hash.new { 1.0 / 0.0 } 
     parents = Hash.new                 

data/lib/rbbt/vector/model/spaCy.rb

@@ -34,10 +34,12 @@ class SpaCyModel < VectorModel
       tmpconfig = File.join(file, 'config')
       tmptrain = File.join(file, 'train.spacy')
       SpaCy.config(@config, tmpconfig)
+
+      bar = bar(features.length, "Training documents into spacy format")
       SpaCyModel.spacy do
         nlp = SpaCy.nlp(lang)
         docs = []
-        RbbtPython.iterate nlp.pipe(texts.zip(labels), as_tuples: true), :bar => "Training documents into spacy format" do |doc,label|
+        RbbtPython.iterate nlp.pipe(texts.zip(labels), as_tuples: true), :bar => bar do |doc,label|
           unique_labels.each do |other_label|
             next if other_label == label
             doc.cats[other_label] = false
@@ -59,16 +61,27 @@ class SpaCyModel < VectorModel
       texts = [texts] unless list
 
       docs = []
+      bar = bar(features.length, "Evaluating model")
       SpaCyModel.spacy do
+        gpu = Rbbt::Config.get('gpu_id', :spacy, :spacy_train, :default => 0)
+        gpu = gpu.to_i if gpu && gpu != ""
+        spacy.require_gpu(gpu) if gpu
         nlp = spacy.load("#{file}/model-best")
 
-        Log::ProgressBar.with_bar texts.length, :desc => "Evaluating documents" do |bar|
-          texts.collect do |text|
-            cats = nlp.(text).cats
-            bar.tick
-            cats.sort_by{|l,v| v.to_f }.last.first
+        docs = nlp.pipe(texts)
+        RbbtPython.collect docs, :bar => bar do |d|
+          Misc.timeout_insist(20) do
+            d.cats.sort_by{|l,v| v.to_f || 0 }.last.first
           end
         end
+        #nlp.(docs).cats.collect{|cats| cats.sort_by{|l,v| v.to_f }.last.first }
+        #Log::ProgressBar.with_bar texts.length, :desc => "Evaluating documents" do |bar|
+        #  texts.collect do |text|
+        #    cats = nlp.(text).cats
+        #    bar.tick
+        #    cats.sort_by{|l,v| v.to_f }.last.first
+        #  end
+        #end
       end
     end
   end

data/lib/rbbt/vector/model/util.rb

@@ -0,0 +1,12 @@
+class VectorModel
+  attr_accessor :bar
+
+  def bar(max = nil, desc = nil)
+    desc, max = max, nil if desc.nil?
+    @bar ||= Log::ProgressBar.new max
+    @bar.desc = desc
+    @bar.max = max
+    @bar.init
+    @bar
+  end
+end

data/lib/rbbt/vector/model.rb

@@ -1,4 +1,5 @@
 require 'rbbt/util/R'
+require 'rbbt/vector/model/util'
 
 class VectorModel
   attr_accessor :directory, :model_file, :extract_features, :train_model, :eval_model

data/share/R/MA.R

@@ -1,4 +1,4 @@
-rbbt.require('limma')
+rbbt.require('edgeR')
 
 #########################################################################
 # Model processing 
@@ -57,6 +57,33 @@ rbbt.dm.matrix.differential.limma.twoside <- function(expr, subset.main, subset.
     return(list(t= fit$t[,2], p.values= fit$p.value[,2]));
 }
 
+rbbt.dm.matrix.differential.DESeq <- function(expr, subset.main, subset.contrast) {
+    rbbt.require('DESeq2')
+    rbbt.require('HTSFilter')
+    rbbt.require('apeglm')
+
+    #expr[expr == 0] = NA
+    good.rows = apply(is.na(expr),1,sum) == 0
+    expr = expr[good.rows,]
+
+    condition_values = rep(c("contrast"), length(subset.contrast))
+    condition_values = c(condition_values, rep(c("condition"), length(subset.main)))
+    names = c(subset.contrast, subset.main)
+    conditions = data.frame(condition = as.factor(condition_values))
+    
+
+    expr = expr[,names]
+
+    dds <- DESeqDataSetFromMatrix(countData = round(expr), colData = conditions, design = ~ condition)
+    dds <- DESeq(dds)
+
+    filter <- HTSFilter(dds, s.len=25, plot=FALSE)$filteredData
+
+    res <- lfcShrink(filter, type="apeglm", coef="condition_contrast_vs_condition")
+
+    return(res)
+}
+
 
 rbbt.dm.matrix.guess.log2 <- function(m, two.channel){
     if (two.channel){
@@ -66,34 +93,24 @@ rbbt.dm.matrix.guess.log2 <- function(m, two.channel){
     }
 }
 
-rbbt.dm.matrix.differential <- function(file, main, contrast = NULL, log2 = FALSE, outfile = NULL, key.field = NULL, two.channel = NULL, namespace = NULL, eBayes.trend = FALSE){
-    if (is.null(namespace)) namespace = rbbt.default_code("Hsa")
-    data = data.matrix(rbbt.tsv(file));
-    dimnames = dimnames(data)
-    original.dimnames = dimnames;
-
-    dimnames[[1]] = make.names(dimnames[[1]])
-    dimnames[[2]] = make.names(dimnames[[2]])
-
-    dimnames(data) <- dimnames
-    main <- make.names(main);
-    contrast <- make.names(contrast);
-
-    data[data == 0] = NA
-    good.rows = apply(is.na(data),1,sum) != dim(data)[2]
-    data = data[good.rows,]
-
-    ids = rownames(data);
-    if (is.null(key.field)){ key.field = "ID" }
-
+rbbt.dm.matrix.differential.limma <- function(data, main, contrast=NULL, log2=NULL, two.channel=NULL, eBayes.trend=NULL){
     if (is.null(log2)){
       log2 = rbbt.dm.matrix.guess.log2(data, two.channel)
     }
 
     if (log2){
-       data = log2(data);
+       cutoff <- 1
+       drop <- which(apply(data, 1, max) < cutoff)
        min = min(data[data != -Inf])
        data[data == -Inf] = min
+       data <- DGEList(data)
+       data <- calcNormFactors(data)
+       data = cpm(data, log=TRUE, prior.count=3)
+       data <- data[-drop,]
+    }else{
+       data[data == 0] = NA
+       good.rows = apply(is.na(data),1,sum) != dim(data)[2]
+       data = data[good.rows,]
     }
 
     if (is.null(contrast)){
@@ -135,20 +152,50 @@ rbbt.dm.matrix.differential <- function(file, main, contrast = NULL, log2 = FALS
 
 
     if (! is.null(limma) && sum(is.na(limma$t)) != length(limma$t)){
+       ids = rownames(data)
        result = data.frame(ratio = ratio[ids], t.values = limma$t[ids], p.values = limma$p.values[ids])
        result["adjusted.p.values"] = p.adjust(abs(result$p.values), "fdr") * sign(result$p.values)
     }else{
        result = data.frame(ratio = ratio)
     }
 
-    rownames(result) <- original.dimnames[[1]][good.rows]
+    rownames(result) <- rownames(data)
+    result = result[!is.na(result$ratio),]
+
+    return(result)
+}
+
+rbbt.dm.matrix.differential <- function(file, main, contrast = NULL, type = 'limma', log2 = FALSE, outfile = NULL, key.field = NULL, two.channel = NULL, namespace = NULL, eBayes.trend = FALSE){
+    data = data.matrix(rbbt.tsv(file));
+    dimnames = dimnames(data)
+
+    original.dimnames = dimnames;
+
+    #dimnames[[1]] = make.names(dimnames[[1]])
+    dimnames[[2]] = make.names(dimnames[[2]])
+
+    dimnames(data) <- dimnames
+    main <- make.names(main);
+
+    if (! is.null(contrast)){
+        contrast <- make.names(contrast);
+    }
+
+    if (type == 'limma')
+        result = rbbt.dm.matrix.differential.limma(data, main, contrast, log2, two.channel, eBayes.trend)
+    else
+        result = rbbt.dm.matrix.differential.DESeq(data, main, contrast)
+
+    if (is.null(outfile)){
+        return(result);
+    }else{
+        if (is.null(key.field)){ key.field = "ID" }
+        if (is.null(namespace)) namespace = rbbt.default_code("Hsa")
+
+        rbbt.tsv.write(outfile, result, key.field, paste(":type=:list#:cast=:to_f#:namespace=", namespace, "#comment=Negative values mark downregulation", sep=""));
+        return(NULL);
+    }
 
-   if (is.null(outfile)){
-       return(result);
-   }else{
-       rbbt.tsv.write(outfile, result, key.field, paste(":type=:list#:cast=:to_f#:namespace=", namespace, "#comment=Negative values mark downregulation", sep=""));
-       return(NULL);
-   }
 }
 
 

data/share/R/barcode.R

@@ -66,17 +66,21 @@ rbbt.GE.barcode.mode <- function(matrix_file, output_file, sd.factor = 2, key.fi
 rbbt.GE.activity_cluster <- function(matrix_file, output_file, key.field = "ID", clusters = c(2,3)){
 
     rbbt.require('mclust')
+    rbbt.require('R.utils')
 
     data = rbbt.tsv.numeric(matrix_file)
 
-    classes = apply(data,1,function(row){
+    classes = apply(data, 1, function(row){
                     row.na = is.na(row)
                     clust = rep(NA, length(row))
-                    if (sum(row.na) <= length(row) - 5){
+                    rbbt.log(str(row))
+                    if (sum(row.na) <= length(row) - 5 && length(unique(row[!row.na])) > 4){
                         clust[!row.na] = densityMclust(row[!row.na], prior=priorControl(), G=clusters)$classification
+                        rbbt.log(str(clust))
                     }
                     clust
     })
+    rbbt.log("DONE")
 
     classes = data.frame(t(classes))
 

data/test/rbbt/network/test_paths.rb

@@ -21,6 +21,26 @@ N4 N5
 
     path = Paths.dijkstra(network, start_node, [end_node])
     assert_equal %w(N1 N2 N4 N5), path.reverse
+
+    path = Paths.dijkstra(network, start_node, end_node)
+    assert_equal %w(N1 N2 N4 N5), path.reverse
+  end
+
+  def test_dijsktra_missing
+    network_txt=<<-EOF
+#: :sep=/\s/#:type=:flat
+#Start End
+N1 N2
+N2 N3 N4
+N4 N5
+    EOF
+    network = TSV.open(StringIO.new(network_txt))
+
+    start_node = "N1"
+    end_node = "M5"
+
+    path = Paths.dijkstra(network, start_node, [end_node])
+    assert_nil path
   end
 
   def test_weighted_dijsktra

data/test/rbbt/vector/model/test_spaCy.rb

@@ -96,14 +96,16 @@ class TestSpaCyModel < Test::Unit::TestCase
 
       model = SpaCyModel.new(
         dir, 
-        "gpu/textcat_accuracy.conf"
+        "cpu/textcat_efficiency.conf"
       )
 
 
+      Rbbt::Config.set 'gpu_id', nil, :spacy
       require 'rbbt/tsv/csv'
       url = "https://raw.githubusercontent.com/hanzhang0420/Women-Clothing-E-commerce/master/Womens%20Clothing%20E-Commerce%20Reviews.csv"
       tsv = TSV.csv(Open.open(url))
       tsv = tsv.reorder("Review Text", ["Recommended IND"]).to_single
+      tsv = tsv.subset(tsv.keys.sample(100))
 
       good = tsv.select("Recommended IND" => '1')
       bad = tsv.select("Recommended IND" => '0')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-dm
 version: !ruby/object:Gem::Version
-  version: 1.1.58
+  version: 1.1.60
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-19 00:00:00.000000000 Z
+date: 2022-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
@@ -111,6 +111,7 @@ files:
 - lib/rbbt/vector/model/spaCy.rb
 - lib/rbbt/vector/model/svm.rb
 - lib/rbbt/vector/model/tensorflow.rb
+- lib/rbbt/vector/model/util.rb
 - share/R/MA.R
 - share/R/barcode.R
 - share/R/heatmap.3.R