rbbt-dm 0.0.1

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2010-2011 Miguel Vázquez García
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/rbbt/statistics/fdr.rb

@@ -0,0 +1,132 @@
+require 'inline'
+
+module FDR
+
+  # values should be sorted
+  def self.step_up_native(values, rate)
+    total = values.length
+
+    last = 0
+    values.each_with_index  do |value, i|
+      if value > rate * (i + 1).to_f / total
+        return last
+      end
+      last = value
+    end
+    return last
+  end
+
+  # values should be sorted
+  def self.adjust_native(values)
+    total = values.length.to_f
+
+    adjusted = []
+    last = 1
+    values.reverse.each_with_index do |value, i|
+      adj = [last, value * total / (total - i )].min
+      last = adj
+      adjusted << adj
+    end
+
+    adjusted.reverse
+  end
+
+  class << self
+    inline do |builder|
+      builder.c <<-EOC
+        double step_up_fast(VALUE ps, double rate){
+           long idx;
+           int total = RARRAY(ps)->len;
+  
+           double last_value = 0;
+           for (idx = 0; idx < total; idx++){
+             double p  = (double) RFLOAT(rb_ary_entry(ps, idx))->value;
+             
+             if (p > rate * (double) (idx + 1) / (double) total){
+                return last_value;
+             }
+             last_value = p;
+           }
+
+           return last_value;
+        }
+
+      EOC
+
+
+      builder.c <<-EOC
+          
+         VALUE adjust_fast_self(VALUE ps){
+           long idx;
+      
+           int total = RARRAY(ps)->len;
+
+           VALUE new = rb_ary_new();
+
+           double last = 1;
+           for (idx = total - 1; idx >= 0 ; idx-- ){
+             double p  = (double) RFLOAT(rb_ary_entry(ps, idx))->value;
+
+      
+             p = p * (double) total / (double) (idx + 1);
+             if (p > last) p = last;
+             last = p;
+
+             RFLOAT(rb_ary_entry(ps, idx))->value = p;
+           }
+
+          return ps;
+         }
+      EOC
+         
+      builder.c <<-EOC
+         VALUE adjust_fast(VALUE ps){
+           long idx;
+      
+           int total = RARRAY(ps)->len;
+
+           VALUE new = rb_ary_new();
+
+           double last = 1;
+           for (idx = total - 1; idx >= 0 ; idx-- ){
+             double p  = (double) RFLOAT(rb_ary_entry(ps, idx))->value;
+
+      
+             p = p * (double) total / (double) (idx + 1);
+             if (p > last) p = last;
+             last = p;
+
+             rb_ary_unshift(new,rb_float_new(p));
+           }
+
+           return new;
+         }
+      EOC
+    end
+  end
+
+  class << self
+    alias_method :adjust, :adjust_fast
+    alias_method :adjust!, :adjust_fast_self
+    alias_method :step_up, :step_up_fast
+  end
+
+  # This will change the values of the floats in situ
+  def self.adjust_hash!(data, field = nil)
+    keys = []
+    values = []
+
+    data.collect{|key, value| [key, field.nil? ? value : value[field]] }.sort{|a,b| 
+      a[1] <=> b[1]
+    }.each{|p|
+      keys << p[0]
+      values << p[1]
+    }
+
+    FDR.adjust!(values)
+
+    data
+  end
+
+end
+

data/lib/rbbt/statistics/hypergeometric.rb

@@ -0,0 +1,146 @@
+require 'inline'
+require 'rbbt/util/tsv'
+
+module Hypergeometric
+  class << self
+    inline do |builder|
+      builder.c_raw <<-EOC
+/**
+ * Compute log(k!)
+ * @param k The value k.
+ * @return The result.
+ */
+double lFactorial(double k)
+{
+	double r = 0;
+	int i;
+	for(i=2 ; i<=(int)k ; i++)
+	{
+		r = r + (double)(log((double)i));
+	}
+	return r;
+}
+
+
+
+/**
+ * Compute the log(binom(n,k))
+ * @param n The number of possible items.
+ * @param k The number of selected items.
+ * @return The result.
+ */
+double lBinom(double n, double k)
+{
+	long i;
+	double r = 0;
+	
+	if(n > n-k){
+		k = n-k;
+	}
+
+	for(i = (long)n ; i> (n-k) ; i--)
+	{
+		r = r + log((double)i);
+	}
+		
+	r = r - lFactorial(k);
+	
+	return r;
+}
+  EOC
+  
+      builder.c <<-EOC
+/**
+*  * Compute the Hypergeometric accumulated value.
+*  * @param total       => total size
+*  * @param support     => total support
+*  * @param list        => selected list size
+*  * @param found       => support
+*  * @return The result
+*  */
+double hypergeometric(double total, double support, double list, double found)
+{
+	double other = total - support;
+
+	double top = list;
+	if(support < list){
+		top = support;
+	}
+
+	double log_n_choose_k = lBinom(total,list);
+
+	double lfoo = lBinom(support,top) + lBinom(other, list-top);
+	
+	double sum = 0;
+  int i;
+	for (i = (int)top; i >= found; i-- )
+	{
+		sum = sum + exp(lfoo - log_n_choose_k);
+		if ( i > found)
+		{
+			lfoo = lfoo + log(i / (support - i+1)) +  log( (other - list + i) / (list-i+1)  );
+		}
+	}
+	return sum;
+}
+      EOC
+    end
+  end
+end
+
+class TSV
+
+  def annotation_counts(fields = nil)
+    fields ||= self.fields
+    fields = [fields] if String === fields or Symbol === fields
+
+    annotation_count_cache_file = TSV.get_persistence_file(File.basename(filename) + "_" + fields.inspect, File.expand_path(File.dirname(filename)))
+
+    if File.exists?(annotation_count_cache_file)
+      Log.low "Loading annotation counts from #{ annotation_count_cache_file }"
+      TCHash.get(annotation_count_cache_file)
+    else
+      Log.low "Saving annotation counts to #{ annotation_count_cache_file }"
+      hash = TCHash.get(annotation_count_cache_file)
+
+      counts = Hash.new(0)
+      through :main, fields do |key, values|
+        values.flatten.compact.uniq.each{|value| counts[value] += 1}
+      end
+      hash.merge! counts
+    end
+  end
+
+  def enrichment(list, fields, options = {})
+    options = Misc.add_defaults options, :min_support => 3
+    Log.debug "Enrichment analysis of field #{fields.inspect} for #{list.length} entities"
+    selected = select :main => list
+    
+    tsv_size = keys.length
+    total = selected.keys.length
+    Log.debug "Found #{total} of #{list.length} entities"
+
+    counts = annotation_counts fields
+
+    annotations = Hash.new 0
+    selected.through :main, fields do |key, values|
+      values.flatten.compact.uniq.each{|value| annotations[value] += 1}
+    end
+
+    pvalues = {}
+    annotations.each do |annotation, count|
+      Log.debug "Hypergeometric: #{ annotation } - #{[tsv_size, counts[annotation], total, count].inspect}"
+      next if count < options[:min_support]
+      pvalue = Hypergeometric.hypergeometric(tsv_size, counts[annotation], total, count)
+      pvalues[annotation] = pvalue
+    end
+
+    FDR.adjust_hash! pvalues if options[:fdr]
+    pvalues.delete_if{|k, pvalue| pvalue > options[:cutoff] } if options[:cutoff]
+
+    pvalues
+  end
+end
+
+
+

data/test/rbbt/statistics/test_fdr.rb

@@ -0,0 +1,41 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+require 'rbbt/statistics/fdr'
+require 'test/unit'
+require 'rsruby'
+
+class TestFDR < Test::Unit::TestCase
+  def clean(values)
+    if Array === values
+      values.collect{|v| (v * 10000).to_i.to_f / 10000}
+    else
+      (values * 10000).to_i.to_f / 10000
+    end
+  end
+
+  def copy(values)
+    values.collect{|v| v + 0.0}
+  end
+
+  def setup
+    @r = RSRuby.instance
+    @values = [0.001, 0.002, 0.003, 0.003, 0.003, 0.004, 0.006, 0.07, 0.09]
+    @threshold = 0.01
+    @r_adj = @r.p_adjust(@values,'BH')
+
+  end
+
+  def test_step_up
+    assert_equal(0.006, clean(FDR.step_up(@values, @threshold)))
+    assert_equal(clean(FDR.step_up_native(@values, @threshold)), clean(FDR.step_up_fast(@values,@threshold)))
+    assert_equal(@r_adj.select{|v| v <= @threshold}.length, @values.select{|v| v <= FDR.step_up(@values, @threshold)}.length)
+  end
+
+  def test_adjust
+    assert_equal(clean(@r_adj), clean(FDR.adjust_native(@values)))
+    assert_equal(clean(FDR.adjust_fast(@values)), clean(FDR.adjust_native(@values)))
+
+    assert_equal(clean(@r_adj), clean(FDR.adjust_fast_self(copy(@values))))
+  end
+end
+
+

data/test/rbbt/statistics/test_hypergeometric.rb

@@ -0,0 +1,45 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../test_helper')
+require 'rbbt/statistics/hypergeometric'
+require 'test/unit'
+
+class TestHypergeometric < Test::Unit::TestCase
+
+  def test_hypergeometric
+    assert Hypergeometric.hypergeometric(100, 20, 15,13) < 0.05
+  end
+
+  def test_annotation_counts
+     content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row3    a    C    Id4
+    EOF
+    
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.new(filename + '#:sep=/\s+/')
+      counts = tsv.annotation_counts
+      assert_equal 2, counts['a']
+    end
+  end
+
+  def test_enrichment
+     content =<<-EOF
+#Id    ValueA    ValueB    OtherID
+row1    a|aa|aaa    b    Id1|Id2
+row2    A    B    Id3
+row3    a    C    Id4
+row4    a    B    Id3
+row5    a    B    Id3
+row6    A    B    Id3
+row7    A    B    Id3
+    EOF
+    
+    TmpFile.with_file(content) do |filename|
+      tsv = TSV.new(filename + '#:sep=/\s+/')
+
+      assert_equal %w(a), tsv.enrichment(%w(row1 row3 row4 row5), "ValueA").collect{|annot,pvalue| pvalue < 0.05 ? annot : nil}.compact
+      assert_equal %w(aa aaa), tsv.enrichment(%w(row1 row3 row4 row5), "ValueA").collect{|annot,pvalue| pvalue > 0.05 ? annot : nil}.compact
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,9 @@
+require 'test/unit'
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+class Test::Unit::TestCase
+  def test_datafile(file)
+    File.join(File.dirname(__FILE__), 'data', file)
+  end
+end

metadata

@@ -0,0 +1,87 @@
+--- !ruby/object:Gem::Specification 
+name: rbbt-dm
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Miguel Vazquez
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-12-20 00:00:00 +01:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rbbt-util
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+description: Data-mining and statistics
+email: miguel.vazquez@fdi.ucm.es
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+files: 
+- LICENSE
+- lib/rbbt/statistics/fdr.rb
+- lib/rbbt/statistics/hypergeometric.rb
+- test/rbbt/statistics/test_fdr.rb
+- test/rbbt/statistics/test_hypergeometric.rb
+- test/test_helper.rb
+has_rdoc: true
+homepage: http://github.com/mikisvaz/rbbt-phgx
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Data-mining and statistics
+test_files: 
+- test/rbbt/statistics/test_fdr.rb
+- test/rbbt/statistics/test_hypergeometric.rb
+- test/test_helper.rb