MARQ 0.0.1

data/lib/MARQ/fdr.rb

@@ -0,0 +1,177 @@
+require 'inline'
+
+module FDR
+
+  # values should be sorted
+  def self.step_up_native(values, rate)
+    total = values.length
+
+    last = 0
+    values.each_with_index  do |value, i|
+      if value > rate * (i + 1).to_f / total
+        return last
+      end
+      last = value
+    end
+    return last
+  end
+
+  # values should be sorted
+  def self.adjust_native(values)
+    total = values.length.to_f
+
+    adjusted = []
+    last = 1
+    values.reverse.each_with_index do |value, i|
+      adj = [last, value * total / (total - i )].min
+      last = adj
+      adjusted << adj
+    end
+
+    adjusted.reverse
+  end
+
+  class << self
+    inline do |builder|
+      builder.c <<-EOC
+        double step_up_fast(VALUE ps, double rate){
+           long idx;
+           int total = RARRAY(ps)->len;
+  
+           double last_value = 0;
+           for (idx = 0; idx < total; idx++){
+             double p  = (double) RFLOAT(rb_ary_entry(ps, idx))->value;
+             
+             if (p > rate * (double) (idx + 1) / (double) total){
+                return last_value;
+             }
+             last_value = p;
+           }
+
+           return last_value;
+        }
+
+      EOC
+
+
+      builder.c <<-EOC
+          
+         VALUE adjust_fast_self(VALUE ps){
+           long idx;
+      
+           int total = RARRAY(ps)->len;
+
+           VALUE new = rb_ary_new();
+
+           double last = 1;
+           for (idx = total - 1; idx >= 0 ; idx-- ){
+             double p  = (double) RFLOAT(rb_ary_entry(ps, idx))->value;
+
+      
+             p = p * (double) total / (double) (idx + 1);
+             if (p > last) p = last;
+             last = p;
+
+             RFLOAT(rb_ary_entry(ps, idx))->value = p;
+           }
+
+          return ps;
+         }
+      EOC
+         
+      builder.c <<-EOC
+         VALUE adjust_fast(VALUE ps){
+           long idx;
+      
+           int total = RARRAY(ps)->len;
+
+           VALUE new = rb_ary_new();
+
+           double last = 1;
+           for (idx = total - 1; idx >= 0 ; idx-- ){
+             double p  = (double) RFLOAT(rb_ary_entry(ps, idx))->value;
+
+      
+             p = p * (double) total / (double) (idx + 1);
+             if (p > last) p = last;
+             last = p;
+
+             rb_ary_unshift(new,rb_float_new(p));
+           }
+
+           return new;
+         }
+      EOC
+    end
+  end
+
+  class << self
+    alias_method :adjust, :adjust_fast
+    alias_method :adjust!, :adjust_fast_self
+    alias_method :step_up, :step_up_fast
+  end
+
+  # This will change the values of the floats in situ
+  def self.adjust_hash!(data, field = nil)
+    keys = []
+    values = []
+
+    data.collect{|key, value| [key, field.nil? ? value : value[field]] }.sort{|a,b| 
+      a[1] <=> b[1]
+    }.each{|p|
+      keys << p[0]
+      values << p[1]
+    }
+
+    FDR.adjust!(values)
+
+    data
+  end
+
+end
+
+if __FILE__ == $0
+
+  require 'test/unit'
+  require 'rsruby'
+
+  class FDRTest < Test::Unit::TestCase
+    def clean(values)
+      if Array === values
+        values.collect{|v| (v * 10000).to_i.to_f / 10000}
+      else
+        (values * 10000).to_i.to_f / 10000
+      end
+    end
+
+    def copy(values)
+      values.collect{|v| v + 0.0}
+    end
+
+    def setup
+      @r = RSRuby.instance
+      @values = [0.001, 0.002, 0.003, 0.003, 0.003, 0.004, 0.006, 0.07, 0.09]
+      @threshold = 0.01
+      @r_adj = @r.p_adjust(@values,'BH')
+
+    end
+
+    def test_step_up
+      assert_equal(0.006, clean(FDR.step_up(@values, @threshold)))
+      assert_equal(clean(FDR.step_up_native(@values, @threshold)), clean(FDR.step_up_fast(@values,@threshold)))
+      assert_equal(@r_adj.select{|v| v <= @threshold}.length, @values.select{|v| v <= FDR.step_up(@values, @threshold)}.length)
+    end
+
+    def test_adjust
+      assert_equal(clean(@r_adj), clean(FDR.adjust_native(@values)))
+      assert_equal(clean(FDR.adjust_fast(@values)), clean(FDR.adjust_native(@values)))
+
+      assert_equal(clean(@r_adj), clean(FDR.adjust_fast_self(copy(@values))))
+    end
+
+  end
+
+  require 'test/unit/ui/console/testrunner'
+  Test::Unit::UI::Console::TestRunner.run(FDRTest)
+
+end

data/lib/MARQ/main.rb

@@ -0,0 +1,227 @@
+require 'MARQ'
+require 'MARQ/MADB'
+require 'MARQ/score'
+
+module MARQ
+  def self.platform_type(platform)
+    if platform.match(/GPL\d+|GDS\d+|GSE\d+/)
+      return :GEO
+    else
+      return :CustomDS
+    end
+  end
+
+  def self.dataset_path(dataset)
+    if platform_type(dataset) == :GEO
+      GEO::dataset_path(dataset)
+    else
+      CustomDS::path(dataset)
+    end
+  end
+
+  def self.platform_organism(platform)
+    if platform_type(platform) == :GEO
+      if platform.match(/^GPL/)
+        GEO::GPL_info(platform)[:organism]
+      else
+        GEO::GPL_info(GEO::dataset_platform(platform))[:organism]
+      end
+    else
+      CustomDS::organism(platform)
+    end
+  end
+
+  def self.organism_platforms(org)
+    GEO::organism_platforms(org) + CustomDS::datasets(org)
+  end
+
+  def self.has_cross_platform?(dataset, platform=nil)
+    if platform_type(platform) == :GEO
+      GEO::has_cross_platform?(dataset, platform)
+    else
+      CustomDS::has_cross_platform?(platform)
+    end
+  end
+
+  def self.is_cross_platform?(dataset, platform=nil)
+    if platform_type(platform) == :GEO
+      GEO::is_cross_platform?(dataset, platform)
+    else
+      CustomDS::is_cross_platform?(platform)
+    end
+  end
+  def self.complete_positions(positions, matched, genes)
+
+    pos = Hash[*matched.zip(positions).flatten]
+    complete = genes.collect{|gene|
+      gene = gene.downcase.strip
+      if matched.include? gene
+        pos[gene] || "MISSING"
+      else
+          "NOT IN PLATFORM"
+      end
+    }
+
+  end
+
+  def self.platform_scores_up_down(platform, up, down)
+    if platform_type(platform) == :GEO
+      GEORQ.platform_scores_up_down(platform, up, down)
+    else
+      CustomDSRQ.scores_up_down(platform, up, down)
+    end
+  end
+
+  module CustomDSRQ
+
+    def self.scores_up_down(platform, up, down)
+      matched_up = DBcache.matches(platform + '_codes', up).length
+      positions_up, matched_up = *MADB::CustomDS.positions(platform, up)
+      missing_up = up.length - matched_up.length
+
+
+
+      matched_down = DBcache.matches(platform + '_codes', down).length
+      positions_down, matched_down = *MADB::CustomDS.positions(platform, down)
+      missing_down = down.length - matched_down.length
+
+      experiments = positions_up.keys
+      platform_entries = MADB::CustomDS.platform_entries(platform)
+
+      scores = experiments.collect{|experiment|
+        score = Score.score_up_down(positions_up[experiment], positions_down[experiment], platform_entries, missing_up, missing_down)
+        score[:total_entries]    = platform_entries
+        score[:positions_up]     = MARQ.complete_positions(positions_up[experiment], matched_up, up) if positions_up.any?
+        score[:positions_down]   = MARQ.complete_positions(positions_down[experiment], matched_down, down) if positions_down.any?
+        score
+      }
+
+      pvalues = Score.pvalues(scores.collect{|s| s[:score]}, up.length, down.length, platform_entries)
+
+      results = {}
+      experiments.each_with_index{|experiment,i|
+        results[experiment] = scores[i].merge(:pvalue => pvalues[i])
+      }
+
+
+      results
+    end
+  end
+
+  module GEORQ
+
+    def self.platform_scores_up_down(platform, up, down)
+      platform_entries = MADB::GEO.platform_entries(platform)
+
+      positions_up, matched_up = *MADB::GEO.positions(platform, up)
+      missing_up = up.length - matched_up.length
+
+      positions_down, matched_down = *MADB::GEO.positions(platform, down)
+      missing_down = down.length - matched_down.length
+
+      return {} if positions_up.keys.empty? && positions_down.keys.empty?
+      if positions_up.keys.any?
+        experiments = positions_up.keys
+      else 
+        experiments = positions_down.keys
+      end
+
+      scores = experiments.collect{|experiment|
+        score = Score.score_up_down(positions_up[experiment], positions_down[experiment], platform_entries, missing_up, missing_down)
+        score[:total_entries]    = platform_entries
+        score[:positions_up]     = MARQ.complete_positions(positions_up[experiment], matched_up, up) if positions_up.any?
+        score[:positions_down]   = MARQ.complete_positions(positions_down[experiment], matched_down, down) if positions_down.any?
+        score
+      }
+
+      pvalues = Score.pvalues(scores.collect{|s| s[:score]}, up.length, down.length, platform_entries)
+
+      results = {}
+      experiments.each_with_index{|experiment,i|
+        results[experiment] = scores[i]
+        results[experiment][:pvalue] = pvalues[i]
+      }
+
+
+      results
+    end
+
+
+    def self.draw_hits(platform, genes, directory)
+      positions, matched = MADB::GEO.positions(platform, genes).values_at(0,1)
+
+      platform_entries = MADB::GEO.platform_entries(platform)
+
+      positions.each{|experiment, positions|
+        Score.draw_hits(positions.compact, platform_entries, File.join(directory, experiment.hash_code + '.png'), {:size => 1000, :bg_color => :green})
+      }
+    end
+
+
+    def self.dataset_positions(dataset, genes)
+      if dataset =~ /_cross_platform/
+        dataset.sub!(/_cross_platform/,'')
+        platform = GEO.dataset_platform(dataset) + '_cross_platform'
+      else
+        platform = GEO.dataset_platform(dataset)
+      end
+
+
+      positions = {}
+      MADB::GEO.positions(platform, genes).first.select{|k,v| k =~ /^#{ dataset }/}.each{|p|
+        positions[p[0].sub(/^#{ dataset }: /,'')] = p[1]
+      }
+      positions
+    end
+
+    def self.dataset_scores(dataset, genes)
+      total = DBcache::num_rows(platform)
+      positions = dataset_positions(dataset, genes)
+
+      scores = {}
+      positions.each{|experiment, positions|
+        scores[experiment] = Score.score(positions, total, genes.length - positions.length)
+      }
+      scores
+    end
+
+  end
+end
+
+if __FILE__ == $0
+  #puts MARQ::organism_platforms('human')
+  #puts MARQ.platform_organism("HaploidData")
+  #puts MARQ::platform_scores_up_down("HaploidData",%w( YMR261c YDL140c YIL122w YPL093w YHR211w YDL142c YHR106w YOR103c YDR233c YLR181c),%w()).keys
+
+  up = %w(
+
+  51228_at    215046_at   205009_at   204915_s_at 202707_at  
+  208265_at   210618_at   201185_at   206650_at   200719_at  
+  215661_at   202071_at   214408_s_at 215092_s_at 206168_at  
+  212686_at   214162_at   221008_s_at 217709_at   210957_s_at
+
+  )
+
+
+  require 'MARQ/ID'
+  require 'pp'
+  genes = ID.translate('human',up).compact
+
+  #pp up.zip(genes)
+  #genes = Open.read("/home/miki/git/MARQ/test/GDS1375_malignant_vs_normal_down.genes").collect{|l| l.chomp.strip}
+  positions =  MARQ::GEORQ.dataset_positions('GDS1231_cross_platform',genes)
+  pp positions
+
+
+  #MARQ::GEORQ.platform_scores_up_down('GPL96_cross_platform',genes,[]).each{|ex, r|
+  #  puts ex
+  #  puts r[:pvalue]
+  #}
+
+  #Score.draw_hits(positions["disease.state: malignant melanoma <=> normal"], MADB::GEORQ.experiment_entries('GPL96','GDS1375: disease.state: malignant melanoma <=> normal') , '/tmp/foo.png',:size => 1000)
+
+
+
+end
+
+

data/lib/MARQ/rankproduct.rb

@@ -0,0 +1,146 @@
+require 'MARQ'
+module RankProduct
+
+  def self.ranks(dataset, experiment , from_FC = false, invert = false)
+    path = MARQ.dataset_path(dataset)
+    codes = Open.read(path + '.codes').collect{|line| line.strip}
+
+    experiments = Open.read(path + '.experiments').collect{|line| line.strip}
+    field = experiments.index experiment.strip
+
+    if from_FC
+      ratios = Open.read(path + '.logratios').collect{|line| 
+        value = line.strip.split("\t")[field]
+        case
+        when value == "NA"
+          nil
+
+        # To sort decreasingly we change sign by default
+        when invert
+          value.to_f
+        else
+          - value.to_f
+        end
+      }
+      Hash[*codes.zip(ratios).sort{|a,b| b[1] <=> a[1]}.collect{|p| p[0]}.zip((1..codes.length).to_a).flatten]
+    else
+      orders = Open.read(path + '.orders').collect{|line| 
+        value = line.strip.split("\t")[field]
+        case
+        when value == "NA"
+          nil
+        when invert
+          codes.length - line.strip.split("\t")[field].to_i + 1
+        else
+          line.strip.split("\t")[field].to_i
+        end
+
+      }
+      Hash[*codes.zip(orders).flatten]
+    end
+  end
+
+  def self.score(gene_ranks, signature_sizes)
+    scores = {}
+    log_sizes = signature_sizes.collect{|size| Math::log(size)}
+    gene_ranks.each{|gene, positions|
+      scores[gene] = positions.zip(log_sizes).
+        collect{|p| Math::log(p[0]) - p[1]}.
+        inject(0){|acc, v| acc += v  }
+    }
+    scores
+  end
+
+  def self.permutations(num_signatures, num = 1000)
+    scores = []
+    num.times{
+       value = 0
+       num_signatures.times{|size_and_log| 
+         value += Math::log(rand)
+       } 
+       scores << value
+    }
+    scores
+  end
+
+  def self.permutations_full(signature_sizes)
+    gene_ranks = {}
+    signature_sizes.each{|size|
+      (1..size).to_a.shuffle.each_with_index{|gene, pos|
+        gene_ranks[gene] ||= []
+        gene_ranks[gene] << pos + 1
+      }
+    }
+    gene_ranks.delete_if{|code, positions| positions.length != signature_sizes.length}
+
+    scores = score(gene_ranks, signature_sizes)
+    scores.values
+  end
+
+  def self.rankproduct(signatures, options = {})
+    invert, from_FC, cross_platform = {
+      :invert => [],
+      :from_FC => false,
+      :cross_platform => false,
+    }.merge(options).values_at(:invert, :from_FC, :cross_platform)
+
+    ranks = {}
+    signatures.each{|signature|
+      dataset, experiment = signature.match(/^([^\:]*): (.*)/).values_at(1,2)
+      dataset = dataset + '_cross_platform' if cross_platform
+      ranks[signature] = self.ranks(dataset, experiment, from_FC, invert.include?(signature))
+    }
+
+    gene_ranks = {}
+    sizes = []
+    ranks.each{|signature, orders|
+      sizes << orders.length
+      orders.each{|code, position|
+        next if position.nil?
+        gene_ranks[code] ||= []
+        gene_ranks[code] << position
+      }
+    }
+
+    gene_ranks.delete_if{|code, positions| positions.length != signatures.uniq.length}
+    
+    scores = score(gene_ranks, sizes)
+    num_permutations = 50000
+
+    permutation_scores = permutations(sizes.length, num_permutations)
+
+    permutation_scores = permutation_scores.sort
+
+
+    results = {}
+    scores.each{|gene, score|
+      pos = permutation_scores.count_smaller(score)
+      results[gene] = [score, pos.to_f / num_permutations]
+    }
+
+
+    num_genes = results.length
+    results.sort{|a,b|
+      a[1][0] <=> b[1][0]
+    }.each_with_index{|p,i|
+      gene = p[0]
+      info = p[1]
+      pvalue = info[1]
+      pfp  = pvalue * num_genes / (i + 1)
+      info << pfp
+    }
+       
+    results
+  end
+
+end
+
+if __FILE__ == $0
+  scores = RankProduct.rankproduct(
+    [ 'GDS2952: disease.state: rheumatoid arthritis <=> normal',
+      #'GDS2952: protocol: pre-treatment <=> untreated',
+      'GDS2952: protocol: post-treatment <=> untreated'], :cross_platform => true, :from_FC => true)
+
+  p scores.sort{|a,b| a[1][0] <=> b[1][0]}[1..10]
+  #scores.sort{|a,b| a[1][0] <=>  b[1][0]}.reverse
+end