rbbt-dm 1.0.0 → 1.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZGQzNzI5ZWRiMzBkMTRjNGIxZmUxYTEwMDAyZDEyZjA1NTI0NTI1NQ==
+  data.tar.gz: !binary |-
+    YTQwYjdlNzU0NGY4ODUzMTBjNDgzMjcxMTAwOWY1OTY2OGM2YjY0Ng==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    ZDFkMTZiMjMxYWE2NjYxZDFlYzFmNjZmMmQzZDhlNWQ5YzhlYzlhNDEzMmI0
+    OGIzYzJkNGFkMjYyMjdjYjVmMjNiNTk4ZTdhYTQzOGJkNjM2MjEwZWJhOTgz
+    YWViMDA3NTY2MDE2NTM0YjRhOWMwYjk2Y2RlMGMxMzU4MjQ2YjE=
+  data.tar.gz: !binary |-
+    MjhmNjc2ODQ5MTZhOWZjNmE0YTQxYzJkN2UzZWQyYjNhZmZkMGJjNjliYTc2
+    NjQwOTk0YjZiMjRmNjFkZjQzZGM3OTAwYmYxZWJhOTM1MTI5ZmNmNzVlNTE3
+    MTE4YThkZmE2ZjVkMjViNzY5ZWI0NmZiMmYwYTY4Y2Y2NDFjMzU=

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2011 Miguel Vázquez García
+Copyright (c) 2010-2013 Miguel Vázquez García
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/lib/rbbt/expression/expression.rb

@@ -0,0 +1,99 @@
+require 'rbbt'
+require 'rbbt/tsv'
+require 'rbbt/GE'
+
+module Expression
+  extend Workflow
+
+  def self.load_matrix(data_file, identifier_file, identifier_format, organism)
+    log :open_data, "Opening data file"
+    data = TSV.open(data_file, :type => :double, :unnamed => true)
+
+    organism ||= data.namespace
+   
+    if not (identifier_file.nil? or identifier_format.nil? or data.key_field == identifier_format)
+
+      case
+      when (fields = (TSV.parse_header(Open.open(identifier_file)).fields) and fields.include?(identifier_format))
+        log :attach, "Adding #{ identifier_format } from #{ identifier_file }"
+        data = data.attach identifier_file, :fields => [identifier_format]
+        log :reorder, "Reordering data fields"
+        data = data.reorder identifier_format, data.fields.dup.delete_if{|field| field == identifier_format}
+      else
+        raise "No organism defined and identifier_format did not match available formats" if organism.nil?
+        require 'rbbt/sources/organism'
+        organism_identifiers = Organism.identifiers(organism)
+        data.identifiers = identifier_file
+        log :attach, "Adding #{ identifier_format } from #{ organism_identifiers }"
+        data = data.attach organism_identifiers, :fields => [identifier_format]
+        log :reorder, "Reordering data fields"
+        data = data.reorder identifier_format, data.fields.dup.delete_if{|field| field == identifier_format}
+        data
+      end
+
+      new_data = TSV.setup({}, :key_field => data.key_field, :fields => data.fields, :type => :list, :cast => :to_f, :namespace => organism, :unnamed => true)
+      log :averaging, "Averaging multiple values"
+      data.with_unnamed do
+        data.through do |key, values|
+          new_data[key] = values.collect{|list| Misc.mean(list.collect{|v| v.to_f})}
+        end
+      end
+
+      data = new_data
+    else
+      log :ready, "Matrix ready"
+    end
+
+    data
+  end
+
+  def self.average_samples(matrix_file, samples)
+    matrix = TSV.open(matrix_file)
+    new = TSV.setup({}, :key_field => matrix.key_field, :fields => matrix.fields, :cast => matrix.cast, :namespace => matrix.namespace)
+    positions = samples.collect{|sample| matrix.identify_field sample}.compact
+    matrix.with_unnamed do
+      matrix.through do |key,values|
+        new[key] = Misc.mean(values.values_at(*positions).compact)
+      end
+    end
+
+    new
+  end
+
+  def self.differential(matrix_file, main, contrast, log2, two_channel)
+    header = TSV.parse_header(Open.open(matrix_file))
+    key_field, *fields = header.all_fields
+    namespace = header.namespace
+
+    main = main & fields
+    contrast = contrast & fields
+
+    if Step === self
+      GE.analyze(matrix_file, main, contrast, log2, path, key_field, two_channel)
+      TSV.open(path, :type => :list, :cast => :to_f, :namespace => namespace)
+    else
+      TmpFile.with_file do |path|
+        GE.analyze(matrix_file, main, contrast, log2, path, key_field, two_channel)
+        TSV.open(path, :type => :list, :cast => :to_f, :namespace => namespace)
+      end
+    end
+  end
+
+  def self.barcode(matrix_file, output_file, factor = 3)
+    GE.barcode(matrix_file, output_file, factor)
+  end
+
+  def self.top_up(diff_file, cutoff = 0.05)
+    TSV.open(diff_file, :cast => :to_f).select("adjusted.p.values"){|p| p > 0 and p < cutoff}
+  end
+
+  def self.top_down(diff_file, cutoff = 0.05)
+    cutoff = -cutoff
+    tsv = TSV.open(diff_file, :cast => :to_f).select("adjusted.p.values"){|p| p < 0 and p > cutoff}
+    tsv.each do |key,values|
+      tsv[key] = values.collect{|v| v.abs}
+    end
+    tsv
+  end
+end
+

data/lib/rbbt/expression/matrix.rb

@@ -0,0 +1,164 @@
+require 'rbbt'
+require 'rbbt/util/misc'
+require 'rbbt/persist'
+require 'rbbt/workflow'
+require 'rbbt/GE/GEO'
+require 'rbbt/statistics/fdr'
+require 'rbbt/expression/expression'
+require 'rbbt/expression/signature'
+
+class Matrix
+  extend Resource
+  self.subdir = "var/matrices"
+  MATRIX_DIR = Matrix.root.find
+
+  def self.geo_matrix_for(gds, key_field = nil, organism = nil)
+    data    = GEO[gds].values.produce.find
+    samples = GEO[gds].samples.produce.find
+
+    dataset_info = GEO[gds]['info.yaml'].produce.yaml
+    platform     = dataset_info[:platform]
+    identifiers  = GEO[platform].codes.produce.find
+
+    log2         = ["count"].include? dataset_info[:value_type]
+
+    Matrix.new(data, identifiers, samples, key_field, organism, log2)
+  end
+
+  attr_accessor :data, :identifiers, :labels, :key_field, :organism, :samples, :log2, :channel
+  def initialize(data, identifiers, labels = nil, key_field = nil, organism = nil, log2 = false, channel = false)
+    data.produce if data.respond_to? :produce
+    @data        = data
+    @samples     = TSV::Parser.new(Open.open(data)).fields
+    @identifiers = identifiers
+    @labels      = TSV.open(labels) unless labels.nil?
+    @key_field   = key_field
+    @log2        = log2
+    @channel     = channel
+    @organism    = organism
+  end
+
+  def matrix_file(path = nil)
+    path ||= Persist.persistence_path(data, {:dir => Matrix::MATRIX_DIR}, {:identifiers => identifiers, :labels => labels, :key_field => key_field, :organism => organism})
+    Persist.persist(data, :tsv, :file => path, :check => [data], :no_load => true) do
+      matrix = Expression.load_matrix(data, identifiers, key_field, organism)
+      matrix = matrix.select(:key => Organism.sanctioned_genes(organism).list) if matrix.key_field == "Ensembl Gene ID"
+      matrix
+    end
+    path
+  end
+
+  def average_samples(samples)
+    path = Persist.persistence_path(matrix_file, {:dir => File.join(Matrix::MATRIX_DIR, 'averaged_samples')}, {:samples => samples})
+    Persist.persist(data, :tsv, :file => path, :no_load => true, :check => [matrix_file]) do
+      Expression.average_samples(matrix_file, samples)
+    end
+    path
+  end
+
+  def find_samples(value, field = nil)
+    labels.select(field){|k,v|
+      Array === v ? v.flatten.include?(value) : v == value
+    }.keys
+  end
+
+  def remove_missing(samples)
+    @samples & samples
+  end
+
+  def average_label(value, field = nil)
+    samples = find_samples(value, field)
+    samples = remove_missing(samples)
+    average_samples(samples)
+  end
+
+  def barcode(path = nil, factor = 2)
+    path ||= Persist.persistence_path(matrix_file, {:dir => File.join(Matrix::MATRIX_DIR, 'sample_differences')}, {:main => main, :contrast => contrast, :log2 => log2, :channel => channel})
+    Persist.persist(data, :tsv, :file => path, :no_load => true, :check => [matrix_file]) do
+      Expression.barcode(matrix_file, path, factor)
+      nil
+    end
+    path
+  end
+
+  def sample_differences(main, contrast)
+    path = Persist.persistence_path(matrix_file, {:dir => File.join(Matrix::MATRIX_DIR, 'sample_differences')}, {:main => main, :contrast => contrast, :log2 => log2, :channel => channel})
+    Persist.persist(data, :tsv, :file => path, :no_load => true, :check => [matrix_file]) do
+      Expression.differential(matrix_file, main, contrast, log2, channel)
+    end
+    path
+  end
+
+  def label_differences(main, contrast = nil, field = nil)
+    all_samples = labels.keys
+    main_samples = find_samples(main, field)
+    if contrast
+      contrast_samples = find_samples(contrast, field)
+    else
+      contrast_samples = all_samples - main_samples
+    end
+
+    main_samples = remove_missing(main_samples)
+    contrast_samples = remove_missing(contrast_samples)
+
+    sample_differences(main_samples, contrast_samples)
+  end
+
+  def signature_set(field, cast = nil)
+    path = Persist.persistence_path(matrix_file, {:dir => File.join(Matrix::MATRIX_DIR, 'signature_set')}, {:field => field, :cast => cast})
+    Persist.persist(data, :tsv, :file => path, :no_load => true, :check => [matrix_file]) do
+      signatures = TSV.open(matrix_file, :fields => [], :type => :list, :cast => cast)
+      labels.values.flatten.uniq.sort.each do |value|
+        begin
+          s = Signature.tsv_field(label_differences(value), field, cast)
+          s.fields = [value]
+          signatures.attach s
+        rescue Exception
+          Log.warn("Signature for #{ value } did not compute")
+        end
+      end
+      signatures
+    end
+    path
+  end
+
+  def random_forest_importance(main, contrast = nil, field = nil, options = {})
+    features = Misc.process_options options, :features
+    features ||= []
+
+    path = Persist.persistence_path(matrix_file, {:dir => File.join(Matrix::MATRIX_DIR, 'random_forest_importance')}, {:main => main, :contrast => contrast, :field => field, :features => features})
+    Persist.persist(data, :tsv, :file => path, :no_load => false, :check => [matrix_file]) do
+      all_samples = labels.keys
+      main_samples = find_samples(main, field)
+      if contrast
+        contrast_samples = find_samples(contrast, field)
+      else
+        contrast_samples = all_samples - main_samples
+      end
+
+
+      main_samples     = remove_missing(main_samples)
+      contrast_samples = remove_missing(contrast_samples)
+
+      TmpFile.with_file do |result|
+        R.run <<-EOF
+library(randomForest);
+orig = rbbt.tsv('#{matrix_file}');
+main = c('#{main_samples * "', '"}')
+contrast = c('#{contrast_samples * "', '"}')
+features = c('#{features * "', '"}')
+
+features = intersect(features, rownames(orig));
+data = t(orig[features, c(main, contrast)])
+data = cbind(data, Class = 0)
+data[main, "Class"] = 1
+
+rf = randomForest(factor(Class) ~ ., data, na.action = na.exclude)
+rbbt.tsv.write(rf$importance, filename='#{ result }', key.field = '#{@key_field}')
+        EOF
+
+        TSV.open(result, :type => :single, :cast => :to_f)
+      end
+    end
+  end
+end

data/lib/rbbt/expression/signature.rb

@@ -0,0 +1,151 @@
+require 'rbbt/util/misc'
+
+module Signature
+
+  def self.setup(hash, options = {})
+    hash.extend Signature
+    hash
+  end
+
+  def self.open(file, field = nil, options = {})
+    options = Misc.add_defaults options, :fields => nil, :cast => :to_f, :type => :single
+
+    options[:fields] ||= [field] if field
+
+    tsv = TSV.open(file, options)
+    tsv.extend Signature
+    tsv
+  end
+
+  def self.tsv_field(tsv, field, cast = nil)
+    tsv = TSV.open(tsv) unless TSV === tsv
+    Signature.setup(tsv.column(field, cast))
+  end
+
+  #{{{ Basic manipulation
+
+  def select(*args, &block)
+    Signature.setup(super(*args, &block))
+  end
+
+  def transform(&block)
+    case
+    when (block_given? and block.arity == 2)
+      self.each do |key, value|
+        self[key] = yield key, value
+      end
+    when (block_given? and block.arity == 1)
+      self.each do |key, value|
+        self[key] = yield value
+      end
+    else
+      raise "Block not given, or arity not 1 or 2"
+    end
+    self
+  end
+
+  def abs
+    transform{|value| value.abs}
+  end
+
+  def log
+    transform{|value| Math.log(value)}
+  end
+
+  def values_over(threshold)
+    entity_options = self.entity_options
+    entity_options[:organism] ||= self.namespace
+    Misc.prepare_entity(self.select{|k,v| v >= threshold}.collect{|k,v| k}, self.key_field, entity_options)
+  end
+
+  def values_under(threshold)
+    entity_options = self.entity_options
+    entity_options[:organism] ||= self.namespace
+    Misc.prepare_entity(self.select{|k,v| v <= threshold}.collect{|k,v| k}, self.key_field, entity_options)
+  end
+
+  #{{{ Rank stuff
+
+  def clean_empty
+    Signature.setup(select{|k,v| v.nil? ? false : (v.respond_to?(:empty) ? !v.empty? : true)}.tap{|s| s.unnamed = true})
+  end
+
+  def sorted
+    OrderedList.setup(clean_empty.sort_by{|elem,v| v}.collect{|elem,v| elem})
+  end
+
+  def ranks
+    ranks = TSV.setup({}, :key_field => self.key_field, :fields => ["Rank"], :cast => :to_i, :type => :single)
+    sorted.each_with_index do |elem, i|
+      ranks[elem] = i
+    end
+    ranks
+  end
+
+  #{{{ Pvalue stuff
+  
+  def significant_pvalues(threshold)
+    entity_options = self.entity_options
+    entity_options[:organism] ||= self.namespace
+    if threshold > 0
+      Misc.prepare_entity(self.select{|k,v| v > 0 and v <= threshold}.collect{|k,v| k}, self.key_field, entity_options)
+    else
+      Misc.prepare_entity(self.select{|k,v| v < 0 and v >= threshold}.collect{|k,v| k}, self.key_field, entity_options)
+    end
+  end
+  def pvalue_fdr_adjust!
+    FDR.adjust_hash! self
+    self
+  end
+
+  def pvalue_score
+    transform{|value| value > 0 ? -Math.log(value + 0.00000001) : Math.log(-value + 0.00000001)}
+  end
+
+  def pvalue_sorted
+    OrderedList.setup(clean_empty.transform{|v| v.to_f}.sort{|a,b| 
+      a = a[1]
+      b = b[1]
+      case
+      when a == b
+        0
+      when (a <= 0 and b >= 0)
+        1
+      when (a >= 0 and b <= 0)
+        -2
+      when a > 0
+        a.abs <=> b.abs
+      else
+        b.abs <=> a.abs
+      end
+    
+    }.collect{|elem,v| elem})
+  end
+
+  def pvalue_sorted_weights
+    sorted = clean_empty.transform{|v| v.to_f}.sort{|a,b| 
+      a = a[1]
+      b = b[1]
+      case
+      when a == b
+        0
+      when (a <= 0 and b >= 0)
+        1
+      when (a >= 0 and b <= 0)
+        -2
+      when a > 0
+        a.abs <=> b.abs
+      else
+        b.abs <=> a.abs
+      end
+    }
+
+    keys = []
+    weights = []
+    sorted.each{|k,v| keys << k; weights << - Math.log(v.abs)}
+
+    OrderedList.setup(keys, weights)
+  end
+
+
+end

data/lib/rbbt/plots/heatmap.rb

@@ -1,4 +1,5 @@
 require 'rbbt/util/R'
+require 'rbbt/util/colorize'
 
 module Heatmap
   def self.heatmap(values, filename, options = {})
@@ -45,14 +46,71 @@ module Heatmap
     scale, take_log, add_to_height, colors = Misc.process_options options, 
       :scale, :take_log, :add_to_height, :colors
 
-    width = 200 + (values.fields.length * 16)
-    height = 200 + (values.length * 16)
+    width = 1200 + (values.fields.length * 100)
+    height = 1000 + (values.length * 50)
     size = [width, height].max
     size = [size, 20000].min
+    width = [size, width].min
+    height = [size, height].min
+
+    take_log = take_log ? "TRUE" : "FALSE"
+    heatmap_script = <<-EOF 
+library(ggplot2);
+rbbt.heatmap('#{filename}', #{ width }, #{ height }, data, take_log=#{take_log});
+    EOF
+
+    values.R heatmap_script
+    
+    filename
+  end
+
+  def self.heatmap3(values, filename, options = {})
+    scale, take_log, add_to_height, colors = Misc.process_options options, 
+      :scale, :take_log, :add_to_height, :colors
+
+    width = 1200 + (values.fields.length * 100)
+    height = 1000 + (values.length * 50)
+    size = [width, height].max
+    size = [size, 2000].min
+    width = [size, width].min
+    height = [size, height].min
+
+    take_log = take_log ? "TRUE" : "FALSE"
+
+    map = options.delete :map
+
+    if map
+      values = values.slice(map.keys)
+      clab = TSV.setup(map.keys, :type => :list, :fields => [], :key_field => map.key_field)
+
+      options[:keys] = []
+      options[:colors] = []
+      map.fields.each do |field|
+        color = Colorize.tsv map.slice(field)
+        clab.add_field field do |k, values|
+          color[k].to_rgb
+        end
+        options[:keys] << "" unless options[:keys].empty?
+        options[:keys].concat map.values.uniq
+        options[:colors] << "#000" unless options[:colors].empty?
+        options[:colors].concat color.values_at(*map.keys).collect{|c| c.to_rgb}.uniq
+      end
+
+      if options[:keys].length > 20 
+        options.delete :keys
+        options.delete :colors
+      end
+
+      options[:ColSideColors] = clab
+    end
+
+    other_params = ", " << options.collect{|k,v| [R.ruby2R(k), R.ruby2R(v)] * "="} * ", " if options.any?
 
     heatmap_script = <<-EOF 
-    library(ggplot2);
+library(ggplot2, quietly = TRUE, warn.conflicts = TRUE);
+source('#{Rbbt.share.R["heatmap.3.R"].find}');
 
+rbbt.heatmap.3('#{filename}', #{ width }, #{ height }, data, take_log=#{take_log}#{other_params});
     EOF
 
     values.R heatmap_script
@@ -60,4 +118,5 @@ module Heatmap
     filename
   end
 
+
 end

data/lib/rbbt/statistics/hypergeometric.rb

@@ -1,7 +1,9 @@
 require 'inline'
 require 'rsruby'
-require 'rbbt/tsv'
+require 'rbbt'
+require 'rbbt/persist'
 require 'rbbt/persist'
+require 'rbbt/tsv'
 require 'rbbt/statistics/fdr'
 require 'rbbt/entity'
 require 'distribution'
@@ -111,52 +113,43 @@ module TSV
     fields = [fields] if String === fields or Symbol === fields
     rename = options.delete :rename
 
-    Persist.persist(filename, :yaml, :fields => fields, :persist => persistence, :prefix => "Hyp.Geo.Counts", :other => { :rename => rename }) do 
-      data ||= Hash.new(0)
+    persistence_path = self.respond_to?(:persistence_path)? self.persistence_path : nil
+    Persist.persist(filename, :yaml, :fields => fields, :persist => persistence, :prefix => "Hyp.Geo.Counts", :other => {:rename => rename, :persistence_path => persistence_path}) do 
+      data ||= {}
 
       with_unnamed do
 
         case type
         when :single
           through :key, fields do |key, value|
-            next if value.nil?
-            data[value] += 1
+            next if value.nil? 
+            data[value] ||= []
+            data[value] << key
           end
         when :double
-          if rename
-            Log.debug("Computing annotation counts with rename: #{rename.values.flatten.compact.uniq.sort * ", "} ")
-            through :key, fields do |key, values|
-              next if values.nil?
-              values.flatten.collect{|elem| rename.include?(elem)? rename[elem] : elem }.compact.uniq.each{|value| data[value] += 1 }
-            end
-          else
-            through :key, fields do |key, values|
-              values.flatten.compact.uniq.each{|value| data[value] += 1}
-            end
+          through :key, fields do |key, values|
+            values.flatten.compact.uniq.each{|value| data[value] ||= []; data[value] << key}
           end
         when :list
           through :key, fields do |key, values|
             next if values.nil?
-            values.compact.uniq.each{|value| data[value] += 1}
+            values.compact.uniq.each{|value| data[value] ||= []; data[value] << key}
           end
         when :flat
-          if rename
-            Log.debug("Computing annotation counts with rename: #{rename.values.flatten.compact.uniq.sort * ", "} ")
-            through :key, fields do |key, values|
-              next if values.nil?
-              values.collect{|elem| rename.include?(elem)? rename[elem] : elem }.compact.uniq.each{|value| data[value] += 1 }
-            end
-          else
-            through :key, fields do |key, values|
-              next if values.nil?
-              values.compact.uniq.each{|value| data[value] += 1}
-            end
+          through :key, fields do |key, values|
+            next if values.nil?
+            values.compact.uniq.each{|value| data[value] ||= []; data[value] << key}
           end
         end
 
       end
 
-      data
+      if rename
+        Log.debug("Using renames during annotation counts")
+        Hash[*data.keys.zip(data.values.collect{|l| l.collect{|e| rename.include?(e)? rename[e] : e }.uniq.length }).flatten]
+      else
+        Hash[*data.keys.zip(data.values.collect{|l| l.uniq.length}).flatten]
+      end
     end
   end
 
@@ -164,6 +157,8 @@ module TSV
     options = Misc.add_defaults options, :skip_missing => true, :background => nil
     background, skip_missing = Misc.process_options options, :background, :skip_missing
 
+    list = list.compact.uniq
+
     if Array === background and not background.empty?
       filter
       add_filter(:key, background)
@@ -174,13 +169,11 @@ module TSV
       end
     end
 
-    list = list.compact.uniq
-
     with_unnamed do
       fields ||= self.fields.first
       options = Misc.add_defaults options, :min_support => 3, :fdr => true, :cutoff => false, :add_keys => true
 
-      add_keys, rename = Misc.process_options options, :add_keys, :rename
+      add_keys, rename, masked = Misc.process_options options, :add_keys, :rename, :masked
 
       Log.debug "Enrichment analysis of field #{fields.inspect} for #{list.length} entities"
 
@@ -194,11 +187,11 @@ module TSV
         total = found
         Log.debug "Using #{ found } as sample size; skipping missing"
       else
-        total = list.uniq.length
+        total = list.length
         Log.debug "Using #{ list.length } as sample size"
       end
 
-      counts = annotation_counts fields, options[:persist], :rename => rename
+      counts = annotation_counts fields, options[:persist], :rename => rename, :masked => masked
 
       annotation_keys = Hash.new
       selected.with_unnamed do
@@ -230,14 +223,14 @@ module TSV
           end
 
         when :flat
-          selected.through :key, fields do |key, values|
+          selected.through do |key, values|
+            next if values.nil?
             values.compact.uniq.reject{|value| value.empty?}.each{|value| 
               value = value.dup
               annotation_keys[value] ||= []
               annotation_keys[value] << key
             }
           end
-
         end
 
       end
@@ -249,6 +242,7 @@ module TSV
 
       pvalues = {}
       annotation_keys.each do |annotation, elems|
+        next if masked and masked.include? annotation
         elems = elems.collect{|elem| rename.include?(elem)? rename[elem] : elem }.compact.uniq if rename
         count = elems.length
         next if count < options[:min_support] or not counts.include? annotation

data/lib/rbbt/statistics/random_walk.rb

@@ -53,6 +53,54 @@ module RandomWalk
     }
     EOC
 
+    builder.c_singleton <<-'EOC'
+    double score_plain_weight(VALUE positions, int total, int missing){
+      int idx;
+
+      int position;
+      double penalty;
+      double max_top, max_bottom;
+      double hit_weights = 0;
+
+      VALUE rel_l = rb_ary_new();
+      VALUE rel_q = rb_ary_new();
+
+      rb_ary_push(rel_q,rb_float_new(0));
+
+      // Rescale positions and accumulate weights
+
+      for (idx = 0; idx < RARRAY_LEN(positions); idx++){
+        position = FIX2INT(rb_ary_entry(positions, idx));
+
+        rb_ary_push(rel_l, rb_float_new((double) position / total));
+
+        hit_weights += 1;
+        rb_ary_push(rel_q, rb_float_new(hit_weights));
+      }
+
+      // Add penalty for missing genes
+      penalty = missing * 1;
+      hit_weights  = hit_weights + penalty;
+
+      // Traverse list and get extreme values of:
+      // Proportion of weight covered - Proportion of hits covered
+
+      max_top = max_bottom = 0;
+      for (idx = 0; idx < RARRAY_LEN(positions); idx++){
+        double top    = RFLOAT_VALUE(rb_ary_entry(rel_q, idx + 1)) / hit_weights -
+                        RFLOAT_VALUE(rb_ary_entry(rel_l, idx));
+        double bottom = - (penalty + RFLOAT_VALUE(rb_ary_entry(rel_q, idx))) / hit_weights +
+                        RFLOAT_VALUE(rb_ary_entry(rel_l, idx));
+
+        if (top > max_top)       max_top    = top;
+        if (bottom > max_bottom) max_bottom = bottom;
+      }
+
+     if (max_top > max_bottom) return max_top;
+     else                      return -max_bottom;
+    }
+    EOC
+
     builder.c_raw_singleton <<-'EOC'
     double fitted_weight(int position, int medium){
         double rel_pos = (double) abs(position - medium) / medium; 
@@ -164,10 +212,17 @@ module RandomWalk
   end
 
   class << self
-    alias score score_fitted_weight
-    alias score_weights score_custom_weights
+    attr_accessor :scoring_method
+
+    def set_scoring(method)
+      scoring_method = method
+      class << self; self end.send(:alias_method, :score, method.to_sym)
+    end
   end
 
+  set_scoring :score_fitted_weight
+
+
   def self.combine(up, down)
     return down if up == 0
     return up if down == 0
@@ -180,6 +235,7 @@ module RandomWalk
     end
   end
 
+  # Two sided
   def self.score_up_down(up, down, total, missing = 0)
     scores_up   = score(up, total, missing)
     scores_down = score(down, total, missing)
@@ -187,18 +243,34 @@ module RandomWalk
     combine(scores_up, scores_down)
   end
 
-  # Two sided
-  def self.permutations(size, total, missing = 0, times = 10000)
+  def self.permutations(size, total, missing = 0, times = 10_000)
     if size == 0
       [0] * times
     else
       (1..times).collect do
-        p = Misc.random_sample_in_range(total, size)
-        score(p.sort, total, missing).abs
+        p = []
+        sample_without_replacement(total, size, p)
+
+        score(p, total, missing).abs
       end
     end
   end
 
+  def self.persisted_permutations(size, total, missing = 0, times = 10_000)
+    repo_file = "/tmp/rw_repo5"
+    repo = Persist.open_tokyocabinet(repo_file, false, :float_array)
+    key = Misc.digest([size, total, missing, times, scoring_method].inspect)
+    if repo[key]
+      repo[key]
+    else
+      p = permutations(size, total, missing, times)
+      repo.write
+      repo[key] = p
+      repo.read
+      repo[key]
+    end
+  end
+
   def self.permutations_up_down(size_up, size_down, total, missing = 0, times = 10000)
     (1..times).collect do
       score_up_down(Array.new(size_up){ (rand * total).to_i }.sort, Array.new(size_down){ (rand * total).to_i }.sort, total, missing).abs
@@ -207,7 +279,8 @@ module RandomWalk
 
   def self.pvalue(permutations, score)
     score = score.abs
-    permutations.inject(0){|acc, per| 
+    permutations.inject(1){|acc, per| 
+
       acc += 1 if per > score
       acc
     }.to_f / permutations.length
@@ -322,37 +395,44 @@ module OrderedList
   def pvalue(set, cutoff = 0.1, options = {})
     set = Set.new(set.compact) unless Set === set
     options = Misc.add_defaults options, :permutations => 10000, :missing => 0
-    permutations, missing = Misc.process_options options, :permutations, :missing
+    permutations, missing, persist_permutations = Misc.process_options options, :permutations, :missing, :persist_permutations
 
     hits = hits(set)
-
+   
     return 1.0 if hits.empty?
 
-    target_score = RandomWalk.score(hits.sort, self.length, 0)
-    target_score_abs = target_score.abs
-
-    max = (permutations.to_f * cutoff).ceil
+    target_score = RandomWalk.score(hits.sort, self.length, missing)
 
-    size = set.length
-    total = self.length
-    better_permutation_score_count = 1
-    if size == 0
-      1.0
+    if persist_permutations
+      permutations = RandomWalk.persisted_permutations(set.length, self.length, missing, permutations)
+      RandomWalk.pvalue(permutations, target_score)
     else
-      (1..permutations).each do
-        p= []
-        RandomWalk.sample_without_replacement(total, size, p)
-
-        permutation_score = RandomWalk.score(p.sort, total, missing).abs
-        if permutation_score.abs > target_score_abs
-          better_permutation_score_count += 1
+      # P-value computation
+      target_score_abs = target_score.abs
+
+      max = (permutations.to_f * cutoff).ceil
+
+      size = set.length
+      total = self.length
+      better_permutation_score_count = 1
+      if size == 0
+        1.0
+      else
+        (1..permutations).each do
+          p= []
+          RandomWalk.sample_without_replacement(total, size, p)
+
+          permutation_score = RandomWalk.score(p.sort, total, missing).abs
+          if permutation_score.abs > target_score_abs
+            better_permutation_score_count += 1
+          end
+
+          return 1.0 if better_permutation_score_count > max
         end
-
-        return 1.0 if better_permutation_score_count > max
+        p = (better_permutation_score_count.to_f + 1) / permutations
+        p = -p if target_score < 0
+        p
       end
-      p = better_permutation_score_count.to_f / permutations
-      p = -p if target_score < 0
-      p
     end
   end
 
@@ -390,7 +470,7 @@ module OrderedList
 
         return 1.0 if better_permutation_score_count > max
       end
-      p = better_permutation_score_count.to_f / permutations
+      p = (better_permutation_score_count.to_f + 1) / permutations
       p = -p if target_score < 0
       p
     end
@@ -400,27 +480,30 @@ end
 module TSV
 
   def self.rank_enrichment_for_list(list, hits, options = {})
-    cutoff = Misc.process_options options, :cutoff
+    cutoff = options[:cutoff]
     list.extend OrderedList
     if cutoff
       list.pvalue(hits, cutoff, options)
     else
-      list.pvalue(hits, options)
+      list.pvalue(hits, nil, options)
     end
   end
 
   def self.rank_enrichment(tsv, list, options = {})
+    masked = options[:masked]
     if tsv.fields
       res = TSV.setup({}, :cast => :to_f, :type => :double, :key_field => tsv.key_field, :fields => ["p-value", tsv.fields.first]) 
     else
       res = TSV.setup({}, :cast => :to_f, :type => :double) 
     end
 
-    tsv.with_monitor do
+    list = list.clean_annotations if list.respond_to? :clean_annotations
+    tsv.with_monitor :desc => "Rank enrichment" do
       tsv.with_unnamed do
         tsv.through do |key, values|
-          pvalue = rank_enrichment_for_list(list, values, options)
-          res[key] = [pvalue, (values.respond_to?(:subset) ? values.subset(list) :  values - list)]
+          next if masked and masked.include? key
+          pvalue = rank_enrichment_for_list(list, values.flatten, options)
+          res[key] = [pvalue, (values.respond_to?(:subset) ? values.subset(list) :  values & list)]
         end
       end
     end
@@ -433,4 +516,17 @@ module TSV
   def rank_enrichment(list, options = {})
     TSV.rank_enrichment(self, list, options)
   end
+
+  def ranks_for(field)
+    ranks = TSV.setup({}, :key_field => self.key_field, :fields => ["Rank"], :type => :single, :cast => :to_i)
+    sort_by(field, true).each_with_index do |k, i|
+      ranks[k] = i
+    end
+
+    ranks.entity_options = entity_options
+    ranks.entity_templates = entity_templates
+    ranks.namespace = namespace
+
+    ranks
+  end
 end

data/lib/rbbt/statistics/rank_product.rb

@@ -5,8 +5,8 @@ module RankProduct
     scores = {}
     log_sizes = signature_sizes.collect{|size| Math::log(size)}
     gene_ranks.each{|gene, positions|
-      scores[gene] = positions.zip(log_sizes).
-        collect{|p| Math::log(p[0]) - p[1]}.    # Take log and substract from size (normalize)
+      scores[gene] = positions.collect{|p| p.nil? or (p.respond_to?(:empty?) and p.empty?) ? signature_sizes.max  : p }.zip(log_sizes).
+        collect{|p| Math::log(p[0]) - p[1]}.   
         inject(0){|acc, v| acc += v  }
     }
     scores
@@ -46,7 +46,7 @@ module TSV
     if block_given?
       scores = fields.collect{|field| tsv.sort_by(field, true, &block)}
     else
-      scores = fields.collect{|field| tsv.sort_by(field, true){|gene,values| tsv.type == :double ? values.first.to_f : value.to_f}}
+      scores = fields.collect{|field| tsv.sort_by(field, true){|gene,values| tsv.type == :single ? values.to_f : values.flatten.first.to_f}}
     end
     positions = {}
     
@@ -61,7 +61,10 @@ module TSV
       end
     end
 
-    score = RankProduct.score(positions, fields.collect{ tsv.size })
+    signature_sizes = fields.collect{|field| slice(field).values.select{|v| v and not (v.respond_to?(:empty?) and v.empty?)}.length} 
+
+    #score = RankProduct.score(positions, fields.collect{ tsv.size })
+    score = RankProduct.score(positions, signature_sizes)
 
     score
   end

data/test/rbbt/statistics/test_hypergeometric.rb

@@ -58,7 +58,6 @@ row7    A    B    Id3
       tsv = TSV.open(filename, :sep => /\s+/)
 
       assert_equal %w(a), tsv.enrichment(%w(row1 row3 row4 row5), "ValueA", :fdr => false, :background => %w(row1 row2 row3 row4 row5 row6 row7)).collect{|annot, values| pvalue = values.first.first.to_f; pvalue < 0.05 ? annot : nil}.compact
-      ddd tsv.enrichment(%w(row3 row4 row5), "ValueA", :fdr => false, :background => %w(row1 row2 row3 row4 row5))
       assert_equal %w(), tsv.enrichment(%w(row1 row3 row4 row5), "ValueA", :fdr => false, :background => %w(row1 row2 row3 row4 row5)).collect{|annot, values| pvalue = values.first.first.to_f; pvalue < 0.05 ? annot : nil}.compact
     end
  

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: rbbt-dm
 version: !ruby/object:Gem::Version
-  version: 1.0.0
-  prerelease: 
+  version: 1.1.0
 platform: ruby
 authors:
 - Miguel Vazquez
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-21 00:00:00.000000000 Z
+date: 2013-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rbbt-util
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: RubyInline
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: priority_queue
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: distribution
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: png
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -99,6 +88,9 @@ extra_rdoc_files:
 - LICENSE
 files:
 - LICENSE
+- lib/rbbt/expression/expression.rb
+- lib/rbbt/expression/matrix.rb
+- lib/rbbt/expression/signature.rb
 - lib/rbbt/network/paths.rb
 - lib/rbbt/plots/bar.rb
 - lib/rbbt/plots/heatmap.rb
@@ -108,42 +100,41 @@ files:
 - lib/rbbt/statistics/rank_product.rb
 - lib/rbbt/vector/model.rb
 - lib/rbbt/vector/model/svm.rb
-- test/rbbt/statistics/test_fdr.rb
 - test/rbbt/statistics/test_hypergeometric.rb
 - test/rbbt/statistics/test_random_walk.rb
+- test/rbbt/statistics/test_fdr.rb
+- test/rbbt/network/test_paths.rb
 - test/rbbt/vector/test_model.rb
 - test/rbbt/vector/model/test_svm.rb
-- test/rbbt/network/test_paths.rb
 - test/test_helper.rb
 homepage: http://github.com/mikisvaz/rbbt-phgx
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Data-mining and statistics
 test_files:
-- test/rbbt/statistics/test_fdr.rb
 - test/rbbt/statistics/test_hypergeometric.rb
 - test/rbbt/statistics/test_random_walk.rb
+- test/rbbt/statistics/test_fdr.rb
+- test/rbbt/network/test_paths.rb
 - test/rbbt/vector/test_model.rb
 - test/rbbt/vector/model/test_svm.rb
-- test/rbbt/network/test_paths.rb
 - test/test_helper.rb