rbbt-marq 2.0.0 → 2.1.0

data/lib/MARQ/ID.rb

@@ -36,12 +36,11 @@ module ID
 
         other  = codes[field]
         next if other.nil? || other == ""
-        
 
         #codes.collect{|c| c.split("|")}.flatten.compact.select{|c| c != ""}.uniq.each{|code|
         other.split("|").each{|code|
           begin
-            DBcache.fast_add(tablename, code.downcase, [native])
+            DBcache.fast_add(tablename, code.strip.downcase, [native])
           rescue
             puts $!.message
           end

data/lib/MARQ/MADB.rb

@@ -8,50 +8,57 @@ module MADB
   # {{{ Saving Positions
   
   # Save the actual data, cross_platform or not
-  def self.save_dataset_instance(dataset, cross_platform)
-    dataset += '_cross_platform' if cross_platform
-    prefix = MARQ::Dataset.path(dataset)
+  def self.save_dataset_instance(dataset)
 
-    # Save codes
-    codes = File.open(prefix  + '.codes').collect{|l| l.chomp.downcase}
-    experiments = File.open(prefix + '.experiments').collect{|l| l.chomp}
-    orders = File.open(prefix + '.orders').collect{|l| values = l.chomp.split(/\t/).collect{|v| v == "NA" ? nil : v.to_i };}
+    # Get info
+    codes        = MARQ::Dataset.codes(dataset);
+    experiments  = MARQ::Dataset.experiments(dataset);
+    orders       = MARQ::Dataset.orders(dataset).values_at(*experiments).transpose;
 
     # Save codes and experiments
     DBcache.save(dataset + '_codes', codes)
     DBcache.save(dataset + '_experiments', experiments)
 
-    # Save orders
+    # Asign orders to codes
     data = {}
     codes.each_with_index{|code,i|
-      data[code.to_sym] = orders[i]
+      data[code] = orders[i]
     }
+
+    # Save orders
     case 
     when codes.length < 65535
       type = "SMALLINT UNSIGNED"
     when codes.length < 16777215
-      type = "MEDIUMIN UNSIGNED"
+      type = "MEDIUMINT UNSIGNED"
     else
       type = "INT UNSIGNED"
     end
-
     DBcache.save(dataset, data, [type] * orders.first.length)
+
   end
 
   # Save dataset, all instances, cross_platform if available.
   def self.save_dataset(dataset)
-    save_dataset_instance(dataset, false)
-    save_dataset_instance(dataset, true) if MARQ::Dataset.has_cross_platform?(dataset)
+    save_dataset_instance(dataset)
+    save_dataset_instance(MARQ::Name.cross_platform(dataset)) if MARQ::Dataset.has_cross_platform?(dataset)
     nil
   end
+
+  def self.save_platform_instance(platform)
+    DBcache.save(platform + '_codes', 
+                 MARQ::Platform.is_cross_platform?(platform) ? 
+                   MARQ::Platform.cross_platform(platform) : 
+                   MARQ::Platform.codes(platform))
+  end
   
   def self.save_platform(platform)
     datasets = MARQ::Platform.datasets(platform).sort
     return if datasets.empty?
 
-    DBcache.save(platform + '_codes', MARQ::Platform.codes(platform))
-    DBcache.save(platform + '_codes', MARQ::Platform.cross_platform(platform)) if MARQ::Platform.has_cross_platform? platform
-
+    save_platform_instance(platform)
+    save_platform_instance(MARQ::Name.cross_platform(platform)) if MARQ::Platform.has_cross_platform?(platform)
+    
     datasets.sort.each do |dataset|
       save_dataset(dataset)
     end
@@ -60,10 +67,9 @@ module MADB
   # {{{ Loading Positions
 
   def self.platform_entries(platform)
-    DBcache.num_rows(platform).to_i
+    DBcache.num_rows(platform + '_codes')
   end
 
-
   def self.load_positions(dataset, genes, platform_entries)
     gene_positions = DBcache.load(dataset, genes)
     data = {}
@@ -73,21 +79,21 @@ module MADB
     experiments = DBcache.load(dataset + '_experiments').sort{|a,b| 
       a[0].to_i <=> b[0].to_i
     }.collect{|p| 
-      MARQ::Dataset.clean(dataset) + ": " + p[1].first
+      MARQ::Name.clean(dataset) + ": " + p[1].first
     }
 
     # Get scale factors (to account for genes missing in the dataset)
     scale = (0..experiments.length - 1).collect{|i| 
       rows = DBcache.num_rows(dataset, "C#{i}"); 
       if rows > 0 
-        platform_entries / rows 
+        platform_entries.to_f / rows 
       else 
         nil 
       end 
     }
 
     # Get experiment positions and scale them
-    experiment_x_gene = gene_positions.values.transpose
+    experiment_x_gene = gene_positions.values_at(*matched).transpose
     experiments.each_with_index{|experiment, i|
       next if scale[i].nil? || experiment_x_gene[i].nil?
       values = experiment_x_gene[i].collect{|v| v.nil? ? nil : (v.to_f * scale[i]).to_i}
@@ -100,8 +106,8 @@ module MADB
   def self.dataset_positions(dataset, genes)
     return [{},[],0] if genes.empty?
 
-    genes = genes.collect{|gene| gene.downcase.strip}
-    platform_entries = platform_entries(dataset + '_codes').to_f
+    genes = genes.collect{|gene| gene.to_s.downcase.strip}
+    platform_entries = platform_entries(dataset)
 
     load_positions(dataset, genes, platform_entries)
   end
@@ -111,7 +117,7 @@ module MADB
     return [{},[],0] if genes.empty?
 
     genes = genes.collect {|gene| gene.downcase.strip }
-    platform_entries = platform_entries(platform).to_f
+    platform_entries = platform_entries(platform)
 
     cross_platform = MARQ::Platform.is_cross_platform? platform
     datasets = MARQ::Platform.datasets(platform).sort
@@ -120,7 +126,7 @@ module MADB
     total_matched = []
 
     datasets.each do |dataset|
-      dataset << '_cross_platform' if cross_platform
+      dataset = MARQ::Name.cross_platform dataset if cross_platform
       data, matched = load_positions(dataset, genes, platform_entries)
       total_data = total_data.merge(data)
       total_matched +=  matched

data/lib/MARQ/annotations.rb

@@ -299,7 +299,7 @@ double hypergeometric(double total, double support, double list, double found)
   def self.dataset_annotations(dataset, type, experiment)
     annotation_dir = File.join(MARQ.datadir, (MARQ::Dataset.is_GEO?(dataset) ? 'GEO' : 'CustomDS'), 'annotations')
 
-    term_file =  File.join(annotation_dir, type, MARQ::Dataset.clean(dataset))
+    term_file =  File.join(annotation_dir, type, MARQ::Name.clean(dataset))
    
     if File.exist? term_file
       @@terms_cache[term_file] ||= YAML::load(File.open(term_file))
@@ -331,9 +331,9 @@ double hypergeometric(double total, double support, double list, double found)
       when side.nil?
         experiment_type = type
       when side == :direct && info[:score] >= 0 || side == :inverse && info[:score] < 0
-        experiment_type += '_up'
+        experiment_type = type + '_up'
       else
-        experiment_type += '_down'
+        experiment_type = type + '_down'
       end
 
       annot[experiment] = dataset_annotations(dataset, experiment_type, name)

data/lib/MARQ/main.rb

@@ -3,21 +3,39 @@ require 'MARQ/MADB'
 require 'MARQ/score'
 
 module MARQ
+  module Name
+    def self.clean(name)
+      name.sub(/_cross_platform/,'') unless name.nil?
+    end
+
+    def self.cross_platform(name)
+      if name =~ /_cross_platform/
+        name
+      else
+        name + "_cross_platform"
+      end
+    end
+
+    def self.is_cross_platform?(name)
+      ! name.match(/_cross_platform$/).nil?
+    end
+
+    def self.is_ratio?(name)
+      ! name.match(/\[ratio\]$/).nil?
+    end
+  end
+
   module Platform
     def self.is_GEO?(platform)
       ! platform.match(/^GPL/).nil?
     end
 
     def self.is_cross_platform?(platform)
-      ! platform.match(/_cross_platform$/).nil?
-    end
-
-    def self.clean(name)
-      name.sub(/_cross_platform/,'') unless name.nil?
+      MARQ::Name.is_cross_platform? platform
     end
 
     def self.path(platform)
-      platform = clean(platform)
+      platform = MARQ::Name.clean(platform)
       if is_GEO? platform
         GEO.platform_path(platform)
       else
@@ -25,6 +43,10 @@ module MARQ
       end
     end
 
+    def self.exists?(platform)
+      path(platform) != nil
+    end
+
     def self.has_cross_platform?(platform)
       File.exists? File.join(path(platform), 'cross_platform')
     end
@@ -38,17 +60,17 @@ module MARQ
     end
 
     def self.codes(platform)
-      platform = clean(platform)
+      platform = MARQ::Name.clean(platform)
       Open.read(File.join(path(platform), 'codes')).scan(/[^\s]+/)
     end
 
     def self.cross_platform(platform)
-      platform = clean(platform)
+      platform = MARQ::Name.clean(platform)
       Open.read(File.join(path(platform), 'cross_platform')).scan(/[^\s]+/)
     end
 
     def self.organism(platform)
-      platform = clean(platform)
+      platform = MARQ::Name.clean(platform)
       if is_GEO? platform
         GEO.platform_organism platform
       else
@@ -57,7 +79,7 @@ module MARQ
     end
 
     def self.process(platform)
-      platform = clean(platform)
+      platform = MARQ::Name.clean(platform)
       if is_GEO? platform
         GEO.process_platform(platform)
       else
@@ -79,10 +101,6 @@ module MARQ
       ! dataset.match(/^(?:GDS|GSE)/).nil?
     end
 
-    def self.clean(name)
-      name.sub(/_cross_platform/,'') if name
-    end
-
     def self.path(platform)
       if is_GEO? platform
         GEO.dataset_path(platform)
@@ -92,21 +110,34 @@ module MARQ
     end
 
     def self.exists?(dataset)
-      ! path(dataset).nil?
+      path = path(dataset)
+      if path.nil?
+        return false
+      else
+        return File.exists?(path + '.orders')
+      end
+    end
+
+    def self.broken?(dataset)
+      path = path(dataset)
+
+      return false if path.nil?
+      
+      if File.exists?(path + '.skip')
+        return true
+      else
+        return false
+      end
     end
 
     def self.is_cross_platform?(dataset)
-      ! dataset.match(/_cross_platform$/).nil?
+      MARQ::Name.is_cross_platform? dataset
     end
 
     def self.has_cross_platform?(dataset)
       File.exists?(path(dataset) + '_cross_platform.orders')
     end
 
-    def self.exists?(dataset)
-      path(dataset) != nil
-    end
-
     def self.info(name)
       begin
         title, description = Open.read(path(name) + '.description').split(/\n--\n/).values_at(0,1)
@@ -159,10 +190,12 @@ module MARQ
     end
 
     def self.read_values_t(dataset, file)
+      experiments = experiments(dataset).reject{|experiment| MARQ::Name.is_ratio? experiment }
+     
+      return {} if experiments.empty?
+
       result = {}
 
-      experiments = experiments(dataset).select{|experiment| experiment !~ /\[ratio\]$/ }
-      return {} if experiments.empty?
       experiments.each{|experiment| result[experiment] = [] }
 
       read_file(dataset, file).split(/\n/).each do |line|
@@ -175,7 +208,7 @@ module MARQ
 
 
     def self.experiments(dataset)
-      read_file(dataset, 'experiments').split(/\n/)
+      read_file(dataset, 'experiments').split(/\n/).collect{|exp| exp.strip }
     end
 
     def self.codes(dataset)
@@ -198,29 +231,40 @@ module MARQ
       read_values_t(dataset, 't')
     end
 
+    def self.codes_for(dataset, type, experiment)
+      codes  = codes(dataset)
+      values = send(type, dataset)[experiment]
+      Hash[*codes.zip(values).reject{|p| p.last.nil? }.flatten]
+    end
+
   end
 
   module RankQuery
     def self.complete_positions(positions, matched, genes)
+      matched = matched.collect{|gene| gene.strip.downcase}
+      genes   = genes.collect{|gene| gene.strip.downcase}
+
       pos = Hash[*matched.zip(positions).flatten]
+
       complete = genes.collect{|gene|
-        gene = gene.downcase.strip
         if matched.include? gene
           pos[gene] || "MISSING"
         else
           "NOT IN PLATFORM"
         end
       }
+      complete
     end
 
 
     def self.position_scores(up, down, positions_up, positions_down, platform_entries, matched_up, matched_down, missing_up, missing_down)
       scores = []
+
       positions_up.keys.each do |experiment|
         score = Score.score_up_down(positions_up[experiment], positions_down[experiment], platform_entries, missing_up, missing_down)
         score[:total_entries]    = platform_entries
-        score[:positions_up]     = complete_positions(positions_up[experiment], matched_up, up) if up.any?
-        score[:positions_down]   = complete_positions(positions_down[experiment], matched_down, down) if down.any?
+        score[:positions_up]     = complete_positions(positions_up[experiment] || [], matched_up, up) if up.any?
+        score[:positions_down]   = complete_positions(positions_down[experiment] || [], matched_down, down) if down.any?
         scores << score
       end
 
@@ -248,12 +292,27 @@ module MARQ
       positions_up, matched_up, platform_entries     = MADB.platform_positions(platform, up)
       missing_up = up.length - matched_up.length
 
+
       positions_down, matched_down                   = MADB.platform_positions(platform, down)
       missing_down = down.length - matched_down.length
 
       position_scores(up, down, positions_up, positions_down, platform_entries, matched_up, matched_down, missing_up, missing_down)
     end
 
+    def self.organism_scores(organism, up, down)
+      platforms = MARQ::Platform.organism_platforms(organism).
+        select  {|p| MARQ::Platform.has_cross_platform? p }.
+        collect {|p| MARQ::Name.cross_platform p }
+
+      total_scores = {}
+      platforms.each do |platform|
+        scores = platform_scores(platform, up, down)
+        total_scores.merge!(scores) 
+      end
+
+      total_scores
+    end
+
   end
 end
 

data/lib/MARQ/rankproduct.rb

@@ -17,7 +17,7 @@ module RankProduct
       orders = MARQ::Dataset.orders(dataset)[experiment.strip]
 
       if invert
-        num_genes = orders.length
+        num_genes = codes.length + 1
         orders.collect! {|pos| pos.nil? ? nil : num_genes - pos }
       end
 
@@ -31,7 +31,7 @@ module RankProduct
     log_sizes = signature_sizes.collect{|size| Math::log(size)}
     gene_ranks.each{|gene, positions|
       scores[gene] = positions.zip(log_sizes).
-        collect{|p| Math::log(p[0]) - p[1]}.
+        collect{|p| Math::log(p[0]) - p[1]}.    # Take log and substract from size (normalize)
         inject(0){|acc, v| acc += v  }
     }
     scores
@@ -70,6 +70,7 @@ module RankProduct
       :cross_platform => false,
     }.merge(options).values_at(:invert, :from_FC, :cross_platform)
 
+    # Gather gene ranks from signatures
     ranks = {}
     signatures.each{|signature|
       dataset, experiment = signature.match(/^([^\:]*): (.*)/).values_at(1,2)
@@ -77,6 +78,7 @@ module RankProduct
       ranks[signature] = self.ranks(dataset, experiment, from_FC, invert.include?(signature))
     }
 
+    # Invert the hash, from signature keys to gene keys
     gene_ranks = {}
     sizes = []
     ranks.each{|signature, orders|
@@ -88,32 +90,36 @@ module RankProduct
       }
     }
 
+    # Remove incomplete genes
     gene_ranks.delete_if{|code, positions| positions.length != signatures.uniq.length}
     
+    # Compute scores
     scores = score(gene_ranks, sizes)
-    num_permutations = 50000
 
+    # Compute permutations
+    num_permutations = 50000
     permutation_scores = permutations(sizes.length, num_permutations)
-
     permutation_scores = permutation_scores.sort
 
 
+    # Compute p-values from permutations
     results = {}
-    scores.each{|gene, score|
+    scores.each {|gene, score|
       pos = permutation_scores.count_smaller(score)
       results[gene] = [score, pos.to_f / num_permutations]
     }
 
-
+    # Complete the information with pfp
     num_genes = results.length
-    results.sort{|a,b|
+    results.sort {|a,b|
       a[1][0] <=> b[1][0]
     }.each_with_index{|p,i|
-      gene = p[0]
       info = p[1]
       pvalue = info[1]
+
       pfp  = pvalue * num_genes / (i + 1)
       info << pfp
+
     }
        
     results