rbbt-marq 1.1.0 → 2.0.0

data/lib/MARQ/MADB.rb

@@ -1,229 +1,133 @@
 require 'MARQ'
 require 'MARQ/GEO'
 require 'MARQ/CustomDS'
+require 'MARQ/main'
 
 module MADB
-  module CustomDS
-
-    def self.save(dataset)
-      prefix = Object::CustomDS.path(dataset)
-
-      codes = File.open(prefix  + '.codes').collect{|l| l.chomp.downcase}
-      
-      DBcache.save(dataset + '_codes', codes)
-
-      experiments = File.open(prefix + '.experiments').collect{|l| l.chomp}
-      orders = File.open(prefix + '.orders').collect{|l| values = l.chomp.split(/\t/).collect{|v| v == "NA" ? nil : v.to_i };}
-
-      data = {}
-      codes.each_with_index{|code,i|
-        data[code.to_sym] = orders[i]
-      }
-      case 
-      when codes.length < 65535
-        type = "SMALLINT UNSIGNED"
-      when codes.length < 16777215
-        type = "MEDIUMIN UNSIGNED"
-      else
-        type = "INT UNSIGNED"
-      end
-
-      DBcache.save(dataset + '_experiments', experiments)
-      DBcache.save(dataset, data, [type] * orders.first.length)
-
-      return unless Object::CustomDS::has_cross_platform?(dataset)
-      dataset = dataset + '_cross_platform'
-      prefix = Object::CustomDS.path(dataset)
-
-      codes = File.open(prefix  + '.codes').collect{|l| l.chomp.downcase}
-      
-      DBcache.save(dataset + '_codes', codes)
-
-      experiments = File.open(prefix + '.experiments').collect{|l| l.chomp}
-      orders = File.open(prefix + '.orders').collect{|l| values = l.chomp.split(/\t/).collect{|v| v == "NA" ? nil : v.to_i };}
-
-      data = {}
-      codes.each_with_index{|code,i|
-        data[code.to_sym] = orders[i]
-      }
-      case 
-      when codes.length < 65535
-        type = "SMALLINT UNSIGNED"
-      when codes.length < 16777215
-        type = "MEDIUMIN UNSIGNED"
-      else
-        type = "INT UNSIGNED"
-      end
-
-      DBcache.save(dataset + '_experiments', experiments)
-      DBcache.save(dataset, data, [type] * orders.first.length)
-      nil
-    end
-   
-    def self.positions(dataset, genes)
-      return [{},[]] if genes.empty?
-      genes = genes.collect{|gene| gene.downcase.strip}
-
-      platform_entries = platform_entries(dataset + '_codes').to_f
-
-      data = {}
-      matched = []
-
-      gene_positions = DBcache.load(dataset, genes)
-      matched ||= gene_positions.keys
-
-      experiments = DBcache.load(dataset + '_experiments').sort{|a,b| 
-        a[0].to_i <=> b[0].to_i
-      }.collect{|p| 
-        Object::GEO::clean(dataset) + ": " + p[1].first
-      }
-
-
-      matched = (matched + gene_positions.keys).uniq
-      scale = (0..experiments.length - 1).collect{|i| 
-        rows = DBcache.num_rows(dataset, "C#{i}"); 
-        if rows > 0 
-          platform_entries / rows 
-        else 
-          nil 
-        end 
-      }
-
-      gene_x_experiment = gene_positions.values
-
-      experiment_x_gene = gene_x_experiment.transpose
-
-      experiments.each_with_index{|experiment, i|
-        next if scale[i].nil? || experiment_x_gene[i].nil?
-        values = experiment_x_gene[i].collect{|v| v.nil? ? nil : (v.to_f * scale[i]).to_i}
-        data[experiment] = values
-      }
-      
-      [data, matched]
-    end
 
-    def self.platform_entries(platform)
-      DBcache.num_rows(platform)
+  # {{{ Saving Positions
+  
+  # Save the actual data, cross_platform or not
+  def self.save_dataset_instance(dataset, cross_platform)
+    dataset += '_cross_platform' if cross_platform
+    prefix = MARQ::Dataset.path(dataset)
+
+    # Save codes
+    codes = File.open(prefix  + '.codes').collect{|l| l.chomp.downcase}
+    experiments = File.open(prefix + '.experiments').collect{|l| l.chomp}
+    orders = File.open(prefix + '.orders').collect{|l| values = l.chomp.split(/\t/).collect{|v| v == "NA" ? nil : v.to_i };}
+
+    # Save codes and experiments
+    DBcache.save(dataset + '_codes', codes)
+    DBcache.save(dataset + '_experiments', experiments)
+
+    # Save orders
+    data = {}
+    codes.each_with_index{|code,i|
+      data[code.to_sym] = orders[i]
+    }
+    case 
+    when codes.length < 65535
+      type = "SMALLINT UNSIGNED"
+    when codes.length < 16777215
+      type = "MEDIUMIN UNSIGNED"
+    else
+      type = "INT UNSIGNED"
     end
-  end
-
 
+    DBcache.save(dataset, data, [type] * orders.first.length)
+  end
 
+  # Save dataset, all instances, cross_platform if available.
+  def self.save_dataset(dataset)
+    save_dataset_instance(dataset, false)
+    save_dataset_instance(dataset, true) if MARQ::Dataset.has_cross_platform?(dataset)
+    nil
+  end
+  
+  def self.save_platform(platform)
+    datasets = MARQ::Platform.datasets(platform).sort
+    return if datasets.empty?
 
+    DBcache.save(platform + '_codes', MARQ::Platform.codes(platform))
+    DBcache.save(platform + '_codes', MARQ::Platform.cross_platform(platform)) if MARQ::Platform.has_cross_platform? platform
 
-  module GEO
-
-    def self.saveGPL(platform)
-      datasets = Object::GEO.platform_datasets(platform).sort
-      return if datasets.empty?
-
-
-      codes = File.open(File.join(Object::GEO.platform_path(platform),'codes')).collect{|l| l.chomp.downcase}
-      
-      DBcache.save(platform, codes)
+    datasets.sort.each do |dataset|
+      save_dataset(dataset)
+    end
+  end
 
-      datasets.sort.each{|dataset|
-        path = Object::GEO.dataset_path(dataset)
-        experiments = File.open(path + '.experiments').collect{|l| l.chomp}
-        orders = File.open(path + '.orders').collect{|l| values = l.chomp.split(/\t/).collect{|v| v == "NA" ? nil : v.to_i };}
+  # {{{ Loading Positions
 
-        data = {}
-        codes.each_with_index{|code,i|
-          data[code.to_sym] = orders[i]
-        }
-        case 
-        when codes.length < 65535
-          type = "SMALLINT UNSIGNED"
-        when codes.length < 16777215
-          type = "MEDIUMINT UNSIGNED"
-        else
-          type = "INT UNSIGNED"
-        end
+  def self.platform_entries(platform)
+    DBcache.num_rows(platform).to_i
+  end
 
-        DBcache.save(dataset + '_experiments', experiments)
-        DBcache.save(dataset, data, [type] * orders.first.length)
-      }
 
+  def self.load_positions(dataset, genes, platform_entries)
+    gene_positions = DBcache.load(dataset, genes)
+    data = {}
+    matched = gene_positions.keys 
+
+    # Get experiments
+    experiments = DBcache.load(dataset + '_experiments').sort{|a,b| 
+      a[0].to_i <=> b[0].to_i
+    }.collect{|p| 
+      MARQ::Dataset.clean(dataset) + ": " + p[1].first
+    }
+
+    # Get scale factors (to account for genes missing in the dataset)
+    scale = (0..experiments.length - 1).collect{|i| 
+      rows = DBcache.num_rows(dataset, "C#{i}"); 
+      if rows > 0 
+        platform_entries / rows 
+      else 
+        nil 
+      end 
+    }
+
+    # Get experiment positions and scale them
+    experiment_x_gene = gene_positions.values.transpose
+    experiments.each_with_index{|experiment, i|
+      next if scale[i].nil? || experiment_x_gene[i].nil?
+      values = experiment_x_gene[i].collect{|v| v.nil? ? nil : (v.to_f * scale[i]).to_i}
+      data[experiment] = values
+    }
+
+    [data, matched, platform_entries]
+  end
+ 
+  def self.dataset_positions(dataset, genes)
+    return [{},[],0] if genes.empty?
 
-      return unless File.exist?(File.join(Object::GEO.platform_path(platform),'cross_platform'))
-      codes = File.open(File.join(Object::GEO.platform_path(platform),'cross_platform')).collect{|l| l.chomp.downcase}
+    genes = genes.collect{|gene| gene.downcase.strip}
+    platform_entries = platform_entries(dataset + '_codes').to_f
 
-      DBcache.save(platform + '_cross_platform', codes)
+    load_positions(dataset, genes, platform_entries)
+  end
 
-      Progress.monitor("Saving #{ platform }")
-      datasets.sort.each{|dataset|
-        path = Object::GEO.dataset_path(dataset)
-        next unless File.exists?(path + '_cross_platform.experiments')
-        experiments = File.open(path + '_cross_platform.experiments').collect{|l| l.chomp}
-        orders = File.open(path + '_cross_platform.orders').collect{|l| values = l.chomp.split(/\t/).collect{|v| v == "NA" ? nil : v.to_i };}
 
-        data = {}
-        codes.each_with_index{|code,i|
-          data[code.to_sym] = orders[i]
-        }
+  def self.platform_positions(platform, genes)
+    return [{},[],0] if genes.empty?
 
-        case 
-        when codes.length < 65535
-          type = "SMALLINT UNSIGNED"
-        when codes.length < 16777215
-          type = "MEDIUMIN UNSIGNED"
-        else
-          type = "INT UNSIGNED"
-        end
+    genes = genes.collect {|gene| gene.downcase.strip }
+    platform_entries = platform_entries(platform).to_f
 
+    cross_platform = MARQ::Platform.is_cross_platform? platform
+    datasets = MARQ::Platform.datasets(platform).sort
 
-        DBcache.save(dataset + '_cross_platform_experiments', experiments)
-        DBcache.save(dataset + '_cross_platform', data, [type] * orders.first.length)
-      }
-    end
+    total_data = {}
+    total_matched = []
 
-    def self.positions(platform, genes)
-      return [{},[]] if genes.empty?
-      genes = genes.collect{|gene| gene.downcase.strip}
-
-      datasets = Object::GEO.platform_datasets(platform).sort
-      platform_entries = platform_entries(platform).to_f
-
-      data = {}
-      matched = nil
-
-      datasets.each{|dataset|
-        dataset += '_cross_platform' if Object::GEO::is_cross_platform?(platform)
-        gene_positions = DBcache.load(dataset, genes)
-        matched ||= gene_positions.keys
-        
-        experiments = DBcache.load(dataset + '_experiments').sort{|a,b| 
-          a[0].to_i <=> b[0].to_i
-        }.collect{|p| 
-          Object::GEO::clean(dataset) + ": " + p[1].first
-        }
-
-        scale = (0..experiments.length - 1).collect{|i| 
-          rows = DBcache.num_rows(dataset, "C#{i}"); 
-          if rows > 0 
-            platform_entries / rows 
-          else 
-            nil 
-          end 
-        }
-        
-        gene_x_experiment = gene_positions.values
-
-        experiment_x_gene = gene_x_experiment.transpose
-
-        experiments.each_with_index{|experiment, i|
-          next if scale[i].nil? || experiment_x_gene[i].nil?
-          values = experiment_x_gene[i].collect{|v| v.nil? ? nil : (v.to_f * scale[i]).to_i}
-          data[experiment] = values
-        }
-      }
-
-      [data, matched]
+    datasets.each do |dataset|
+      dataset << '_cross_platform' if cross_platform
+      data, matched = load_positions(dataset, genes, platform_entries)
+      total_data = total_data.merge(data)
+      total_matched +=  matched
     end
+    total_matched.uniq!
 
-    def self.platform_entries(platform)
-      DBcache.num_rows(platform)
-    end
+    [total_data, total_matched, platform_entries]
   end
 
 end
@@ -232,7 +136,8 @@ if __FILE__ == $0
   #CustomDS::datasets('sgd').each{|d| MADB::CustomDS::save(d)}
 
   require 'pp'
-  pp MADB::GEO::positions('GPL91_cross_platform', %w(2778))[0].select{|k,v| k =~ /GDS989/}.sort
+  pp MADB::dataset_positions('GDS113', %w(2778))[0]
+  pp MADB::platform_positions('GPL54', %w(2778))[0]
   #p MADB::CustomDS::positions("HaploidData",%w( YMR261c YDL140c YIL122w YPL093w YHR211w YDL142c YHR106w YOR103c YDR233c YLR181c yomeman))
   #p MADB::CustomDS::positions("HaploidData_cross_platform",%w( S000002685 S000001149 S000003068 S000003153 S000003355 S000000127 S000004444 S000004875 S000001702 S000005843 S000000862))
 end

data/lib/MARQ/annotations.rb

@@ -2,8 +2,12 @@ require 'inline'
 require 'net/http'
 require 'uri'
 require 'MARQ'
-require 'rbbt/bow/dictionary'
 require 'MARQ/fdr'
+require 'digest/md5'
+require 'base64'
+require 'rbbt/util/open'
+require 'MARQ/main'
+require 'rbbt/bow/dictionary'
 
 module Annotations
   class << self
@@ -291,18 +295,25 @@ double hypergeometric(double total, double support, double list, double found)
   end
 
 
+  @@terms_cache = {}
+  def self.dataset_annotations(dataset, type, experiment)
+    annotation_dir = File.join(MARQ.datadir, (MARQ::Dataset.is_GEO?(dataset) ? 'GEO' : 'CustomDS'), 'annotations')
+
+    term_file =  File.join(annotation_dir, type, MARQ::Dataset.clean(dataset))
+   
+    if File.exist? term_file
+      @@terms_cache[term_file] ||= YAML::load(File.open(term_file))
+      terms = @@terms_cache[term_file] 
+      {:dataset => (terms[:dataset] || []), :signature => (terms[experiment] || [])}
+    else
+      {:dataset =>  [], :signature => []}
+    end
+  end
 
   def self.annotations(scores, type, pvalue = 0.05, algorithm = :rank) 
     annot = {}
     relevant = []
 
-    dict_options = {}
-    if type == "Words"
-      dict_options = {:low => 0, :hi => 0.05, :limit => 100000}
-    else
-      dict_options = {:low => 0, :hi => 0.5, :limit => 100000}
-    end
-
     case
     when type =~ /^(.*)_direct$/
       side = :direct
@@ -312,31 +323,31 @@ double hypergeometric(double total, double support, double list, double found)
       type = $1
     end
 
-
-    terms_cache = {}
     scores.each{|experiment, info|
       dataset = experiment.match(/^(.*?): /)[1]
       name = $'.strip 
+      
       case
       when side.nil?
-        term_file = File.join(MARQ.datadir, MARQ.platform_type(dataset).to_s , 'annotations',type, dataset)
-      when side == :direct && info[:score] > 0 || side == :inverse && info[:score] < 0
-        term_file = File.join(MARQ.datadir, MARQ.platform_type(dataset).to_s , 'annotations',type + '_up', dataset)
+        experiment_type = type
+      when side == :direct && info[:score] >= 0 || side == :inverse && info[:score] < 0
+        experiment_type += '_up'
       else
-        term_file = File.join(MARQ.datadir, MARQ.platform_type(dataset).to_s , 'annotations',type + '_down', dataset)
+        experiment_type += '_down'
       end
 
-      if File.exist? term_file
-        terms_cache[term_file] ||= YAML::load(File.open(term_file))
-        terms = terms_cache[term_file] 
-        annot[experiment] = {:dataset => (terms[:dataset] || []), :signature => (terms[name] || [])}
-      else
-        annot[experiment] = {:dataset =>  [], :signature => []}
-      end
+      annot[experiment] = dataset_annotations(dataset, experiment_type, name)
 
       relevant << experiment if info[:pvalue] <= pvalue
     }
 
+    dict_options = {}
+    if type == "Words"
+      dict_options = {:low => 0, :hi => 0.05, :limit => 100000}
+    else
+      dict_options = {:low => 0, :hi => 0.5, :limit => 100000}
+    end
+
     if algorithm == :rank
       ranks = scores.sort{|a,b| compare(a[1],b[1]) }.collect{|p| p[0]}
       terms = enrichment_rank(annot, ranks, dict_options)
@@ -348,10 +359,11 @@ double hypergeometric(double total, double support, double list, double found)
     annot.each{|key, info|
       merged_annotations[key] = info[:dataset] + info[:signature]
     }
+
     [merged_annotations, terms]
   end
 
-  module GO
+  module Genes
     module Genecodis
       ORGS = {
         'sgd' => 'Sc' ,
@@ -435,8 +447,82 @@ double hypergeometric(double total, double support, double list, double found)
       end
     end
 
+    module SENT
+
+      class SENTError < StandardError; end
+
+
+      WSDL="http://sent.dacya.ucm.es/wsdl/SentWS.wsdl"
+      def self.driver
+        require 'soap/wsdlDriver'
+        driver = SOAP::WSDLDriverFactory.new(WSDL).create_rpc_driver
+        driver
+      end
+
+      def self.process_results(job)
+        result_ids = driver.results(job)
+
+        summary = YAML::load(Base64.decode64(driver.result(result_ids[0])))
+        ccc     = Base64.decode64(driver.result(result_ids[1])).to_f
+        associations = Open.to_hash(StringIO.new(driver.associations(job)), :flatten => true)
+
+        summary.each do |group|
+          group[:articles] = group[:genes].inject(0) {|acc, gene| acc += associations[gene].length}
+        end
+
+        [summary, ccc]
+      end
+
+      @@jobs = {}
+      def self.analyze(organism, genes, factors)
+        hash = Digest::MD5.hexdigest([organism, genes.sort].inspect)
+
+        if @@jobs[hash]
+          orig_job = @@jobs[hash]
+          job = driver.refactor(orig_job, factors, 'MARQ')
+        else
+          job = driver.analyze(organism, genes, factors, 'MARQ')
+          orig_job = job
+        end
+
+        puts "#{ job }: #{ factors }"
+
+        while ! driver.done(job)
+          sleep 5
+        end
+
+        raise SENT::SENTError, "Job failed with error #{driver.messages(job).last}" if driver.error(job)
+        @@jobs[hash] = job
+
+        summary, ccc = process_results(orig_job)
+      end
+
+      def self.terms(organism, genes, num = 20)
+        factor_list = [2,4,8,10]
+        
+        terms = {}
+        cccs   = {}
+        factor_list.each do |factors|
+          summary, ccc = analyze(organism, genes, factors)
+          articles = summary.inject(0) {|acc, group| acc += group[:articles] }
+          terms_per_article = num.to_f / articles
+          summary.each{|group|
+            num_terms = [terms_per_article * group[:articles], group[:words].length].min
+            terms[factors] ||= []
+            terms[factors] += group[:words][0..(num_terms - 1)]
+            p terms
+          }
+          cccs[factors] = ccc
+        end
+
+        best_k = cccs.sort_by{|p| p[1]}.first[1]
+
+        terms[k]
+      end
+    end
+
     def self.get_genes_nth(dataset, num_genes)
-      path = MARQ.dataset_path(dataset)
+      path = MARQ::Dataset.path(dataset)
 
       experiments = File.open(path + '.experiments').collect{|l| l.chomp.strip}
       genes       = File.open(path + '.codes').collect{|l| l.chomp.strip}
@@ -472,7 +558,7 @@ double hypergeometric(double total, double support, double list, double found)
       if nth_genes > 0
         return get_genes_nth(dataset, nth_genes)
       end
-      
+
 
       path = MARQ.dataset_path(dataset)
 
@@ -509,7 +595,7 @@ double hypergeometric(double total, double support, double list, double found)
           values_down[name]   << (p.last < 0 ? - value : 0)
         }
       }
-     
+
       genes_up = {}
       genes_down = {}
 
@@ -545,7 +631,7 @@ double hypergeometric(double total, double support, double list, double found)
 
     def self.get_genes_old(dataset, cut_off = 0.1, fdr = false)
 
-      path = MARQ.dataset_path(dataset)
+      path = MARQ::Dataset.path(dataset)
 
       experiments = File.open(path + '.experiments').collect{|l| l.chomp.strip}.select{|name| !name.match(/\[ratio\]/)}
       genes = File.open(path + '.codes').collect{|l| l.chomp.strip}
@@ -598,7 +684,7 @@ double hypergeometric(double total, double support, double list, double found)
     def self.OBA(text)
 
       res = Net::HTTP.post_form(URI.parse('http://rest.bioontology.org/obs_hibernate/annotator'),
-        {
+                                {
           'longestOnly'=> true, 
           'wholeWordOnly'=> true,
           'withDefaultStopWords' => true,
@@ -626,11 +712,11 @@ end
 if __FILE__ == $0
   require 'pp'
 
-  
-  
+
+
   exit
   #Annotations::GO::Genecodis::Local.init
- 
+
   #genes = Annotations::GO::get_genes('GDS1916')
   #genes[:up].each{|exp, genes|
   #  puts exp
@@ -647,15 +733,15 @@ if __FILE__ == $0
   and 24 hours following treatment with 4 mg/kg body weight GH. Results provide
   insight into the insulin-like growth factor-I dependent and independent
   pathways that mediate the action of GH in bone.
-  
+
   EOT
   texts << <<-EOT
-  
+
   Comparison of total transcription profiles for temperature-sensitive TOR2
   mutant strain SH121 to its isogenic wild type counterpart SH100. Results
   indicate that TOR2 inactivation leads to enhanced transcription of
   Gcn4-controlled target genes.
-  
+
 
   EOT
   texts << <<-EOT
@@ -664,8 +750,8 @@ if __FILE__ == $0
   melanoma in situ, vertical growth phase (VGP) melanoma, and metastatic growth
   phase (MGP) melanoma. Results identify expression signatures that distinguish
   benign and atypical nevi and melanomas in situ from VGPs and MGPs.
-  
-  
+
+
   EOT
   texts << <<-EOT
 
@@ -678,7 +764,7 @@ if __FILE__ == $0
   EOT
   texts << <<-EOT
 
-  
+
   Analysis of anaerobic chemostat cultures of Saccharomyces cerevisae exposed
   to one of several weak organic acids. Weak organic acids are used as
   preservatives in food and beverages. Yeasts are able to proliferate at the
@@ -721,7 +807,7 @@ if __FILE__ == $0
   key component of host defense. Results suggest most of the host response to
   endotoxin or live bacteria is actually regulated independently of MyD88.
 
-  
+
   EOT
   texts.reverse.each{|text|
     puts "\n\n--------------\n"
@@ -732,7 +818,7 @@ if __FILE__ == $0
     puts Annotations::UMLS::OBA(text).join(", ")
   }
 
-#
+  #
 
   #puts  Annotations.hypergeometric(2000,100,100,2)
   #p Annotations::GO::annotate(MARQ.platform_organism('GDS1365'),genes[:up].collect.first.last[1..100])