MARQ 0.0.1

data/lib/MARQ/ID.rb

@@ -0,0 +1,148 @@
+require 'MARQ'
+require 'DBcache'
+
+module ID
+
+  #DEFAULT_FORMAT_ALL = 'Entrez Gene Id'
+  DEFAULT_FORMAT_ALL = nil
+  DEFAULT_FORMATS = {}
+
+  def self.id_position(org, id)
+    @@supported[org] ||= Organism.supported_ids(org)
+    Organism.id_position(@@supported[org], id)
+  end
+
+
+  def self.DB_load(org, to = 0)
+    identifier_file = File.join(Rbbt.datadir,'organisms',org,'identifiers')
+    tablename = "ID_#{org.to_s.strip}_#{to.to_s.strip}"
+    
+    if DBcache.has_table?(tablename)
+      DBcache.drop(tablename)
+    end
+
+    DBcache.create(tablename, 'CHAR(50)', ['CHAR(50)'])
+
+    require 'progress-monitor'
+
+    total_fields = Organism.supported_ids(org).length
+    
+    Progress.monitor "Loading ID DB table for #{ org } - #{to}"
+    total_fields.times{|field|
+      Progress.monitor "Processing file. Field #{ field } / #{total_fields}"
+      File.open(identifier_file).each{|l|
+        next if l =~ /^#/
+        codes = l.chomp.split(/\t/)
+
+        native = codes[to]
+        next if native.nil? || native == "" 
+
+        other  = codes[field]
+        next if other.nil? || other == ""
+        
+
+        #codes.collect{|c| c.split("|")}.flatten.compact.select{|c| c != ""}.uniq.each{|code|
+        other.split("|").each{|code|
+          begin
+            DBcache.fast_add(tablename, code.downcase, [native])
+          rescue
+            puts $!.message
+          end
+        }
+      }
+    }
+  end
+
+  def self.translate_DB(org, genes, options = {})
+    to = options[:to] || DEFAULT_FORMATS[org] || DEFAULT_FORMAT_ALL
+
+    if to
+      to = id_position(org, to)
+    else 
+      to = 0
+    end
+
+    tablename = "ID_#{org.to_s.strip}_#{to.to_s.strip}"
+    DB_load(org, to) unless DBcache.has_table?(tablename)
+    genes = genes.collect{|gene| gene.strip.downcase}
+    DBcache.load(tablename, genes).values_at(*genes).collect{|gene| gene.first if gene}
+  end
+  
+  def self.translate_index(org, genes, options = {})
+    genes = genes.collect{|gene| gene.strip if gene}
+    to = options[:to] || DEFAULT_FORMATS[org] || DEFAULT_FORMAT_ALL
+    from = options[:from]
+    @indexes ||= {}
+    if  @indexes[org.to_s + to.to_s + from.to_s].nil?
+      puts "Loading #{ org }"
+      options = {}
+      options[:other] = [from] if from
+      options[:native] = to if to
+      options[:case_sensitive] = false
+      
+      @indexes[org.to_s + to.to_s + from.to_s] = Organism.id_index(org, options)
+    end
+
+    index =  @indexes[org.to_s + to.to_s + from.to_s]
+    
+    return genes.collect{|code| 
+      code.nil? || code == "" ? nil : index[code]
+    }
+  end
+
+  @@supported = {}
+  def self.translate_grep(org, genes, options = {})
+    to = options[:to] || DEFAULT_FORMATS[org] || DEFAULT_FORMAT_ALL
+    from = options[:from]
+
+    options = {}
+    options[:extra]  = [id_position(org, from)] if from
+    options[:native] = id_position(org,to) if to
+    options[:case_sensitive] ||= false
+
+    genes = genes.collect{|gene| gene.strip if gene}
+    genes = genes.collect{|gene| gene.downcase} unless options[:case_sensitive] 
+
+    #genes_re = '\(^\|[' + "\t" + '|]\)' + genes.join('\|') + '\($\|[' + "\t" + '|]\)'
+    #cmd = "cat #{File.join(Rbbt.datadir,'organisms',org,'identifiers')}|grep -i '#{genes_re}'"
+    genes_re = '(?:^|[\t\|])(?:' + genes.join('|') + ')(?:$|[\t\|])'
+    cmd = "cat #{File.join(Rbbt.datadir,'organisms',org,'identifiers')}|ruby -lne 'puts $_ if $_ =~ /#{genes_re}/i'"
+
+    index = Index.index(IO::popen(cmd), options)
+
+    index.values_at(*genes)
+
+  end
+
+
+  def self.AILUN_translate(platform, genes)
+    index = Open.to_hash("ftp://ailun.stanford.edu/ailun/annotation/geo/#{platform}.annot.gz", :fix => proc{|l| l.match(/^(.*?)\t(.*?)\t.*/); $1.downcase + "\t" + $2 })
+    index.values_at(*genes.collect{|code| code.strip.downcase}).collect{|v| v.nil? ? nil : v.first.first}
+  end
+
+  class << self
+    alias_method :translate, :translate_DB
+  end
+
+end
+
+if __FILE__ == $0
+  require 'benchmark'
+
+
+  Organism.all.each{|org|
+    ID.DB_load(org)
+  }
+
+
+  #num = 4000
+
+  #genes = File.open(Rbbt.datadir + '/organisms/human/identifiers').collect{|l| l.split("\t")[2] }.select{|c| c != ""}[1000,num]
+
+  #trans = nil
+  #puts Benchmark.measure{
+  #  trans = ID.translate('human', genes)
+  #}
+  #p trans
+  #
+end

data/lib/MARQ/MADB.rb

@@ -0,0 +1,243 @@
+require 'MARQ'
+require 'MARQ/GEO'
+require 'MARQ/CustomDS'
+
+module MADB
+  module CustomDS
+
+    def self.save(dataset)
+      prefix = Object::CustomDS.path(dataset)
+
+      codes = File.open(prefix  + '.codes').collect{|l| l.chomp.downcase}
+      
+      DBcache.save(dataset + '_codes', codes)
+
+      experiments = File.open(prefix + '.experiments').collect{|l| l.chomp}
+      orders = File.open(prefix + '.orders').collect{|l| values = l.chomp.split(/\t/).collect{|v| v == "NA" ? nil : v.to_i };}
+
+      data = {}
+      codes.each_with_index{|code,i|
+        data[code.to_sym] = orders[i]
+      }
+      case 
+      when codes.length < 65535
+        type = "SMALLINT UNSIGNED"
+      when codes.length < 16777215
+        type = "MEDIUMIN UNSIGNED"
+      else
+        type = "INT UNSIGNED"
+      end
+
+      puts "Saving #{ dataset }"
+      DBcache.save(dataset + '_experiments', experiments)
+      DBcache.save(dataset, data, [type] * orders.first.length)
+
+      return unless Object::CustomDS::has_cross_platform?(dataset)
+      dataset = dataset + '_cross_platform'
+      prefix = Object::CustomDS.path(dataset)
+
+      codes = File.open(prefix  + '.codes').collect{|l| l.chomp.downcase}
+      
+      DBcache.save(dataset + '_codes', codes)
+
+      experiments = File.open(prefix + '.experiments').collect{|l| l.chomp}
+      orders = File.open(prefix + '.orders').collect{|l| values = l.chomp.split(/\t/).collect{|v| v == "NA" ? nil : v.to_i };}
+
+      data = {}
+      codes.each_with_index{|code,i|
+        data[code.to_sym] = orders[i]
+      }
+      case 
+      when codes.length < 65535
+        type = "SMALLINT UNSIGNED"
+      when codes.length < 16777215
+        type = "MEDIUMIN UNSIGNED"
+      else
+        type = "INT UNSIGNED"
+      end
+
+      puts "Saving #{ dataset }"
+      DBcache.save(dataset + '_experiments', experiments)
+      DBcache.save(dataset, data, [type] * orders.first.length)
+      nil
+    end
+   
+    def self.positions(dataset, genes)
+      return [{},[]] if genes.empty?
+      genes = genes.collect{|gene| gene.downcase.strip}
+
+      platform_entries = platform_entries(dataset + '_codes').to_f
+
+      data = {}
+      matched = []
+
+      gene_positions = DBcache.load(dataset, genes)
+      matched ||= gene_positions.keys
+
+      experiments = DBcache.load(dataset + '_experiments').sort{|a,b| 
+        a[0].to_i <=> b[0].to_i
+      }.collect{|p| 
+        Object::GEO::clean(dataset) + ": " + p[1].first
+      }
+
+
+      matched = (matched + gene_positions.keys).uniq
+      scale = (0..experiments.length - 1).collect{|i| 
+        rows = DBcache.num_rows(dataset, "C#{i}"); 
+        if rows > 0 
+          platform_entries / rows 
+        else 
+          nil 
+        end 
+      }
+
+      gene_x_experiment = gene_positions.values
+
+      experiment_x_gene = gene_x_experiment.transpose
+
+      experiments.each_with_index{|experiment, i|
+        next if scale[i].nil? || experiment_x_gene[i].nil?
+        values = experiment_x_gene[i].collect{|v| v.nil? ? nil : (v.to_f * scale[i]).to_i}
+        data[experiment] = values
+      }
+      
+      [data, matched]
+    end
+
+    def self.platform_entries(platform)
+      DBcache.num_rows(platform)
+    end
+  end
+
+
+
+
+
+  module GEO
+
+    require 'progress-monitor'
+    def self.saveGPL(platform)
+      datasets = Object::GEO.platform_datasets(platform).sort
+      return if datasets.empty?
+
+      puts "Saving #{ platform }, #{datasets.length} datasets"
+
+      codes = File.open(File.join(Object::GEO.platform_path(platform),'codes')).collect{|l| l.chomp.downcase}
+      
+      DBcache.save(platform, codes)
+
+      Progress.monitor("Saving #{ platform }")
+      datasets.sort.each{|dataset|
+        path = Object::GEO.dataset_path(dataset)
+        experiments = File.open(path + '.experiments').collect{|l| l.chomp}
+        orders = File.open(path + '.orders').collect{|l| values = l.chomp.split(/\t/).collect{|v| v == "NA" ? nil : v.to_i };}
+
+        data = {}
+        codes.each_with_index{|code,i|
+          data[code.to_sym] = orders[i]
+        }
+        case 
+        when codes.length < 65535
+          type = "SMALLINT UNSIGNED"
+        when codes.length < 16777215
+          type = "MEDIUMINT UNSIGNED"
+        else
+          type = "INT UNSIGNED"
+        end
+
+        DBcache.save(dataset + '_experiments', experiments)
+        DBcache.save(dataset, data, [type] * orders.first.length)
+      }
+
+
+      return unless File.exist?(File.join(Object::GEO.platform_path(platform),'cross_platform'))
+      codes = File.open(File.join(Object::GEO.platform_path(platform),'cross_platform')).collect{|l| l.chomp.downcase}
+
+      DBcache.save(platform + '_cross_platform', codes)
+
+      Progress.monitor("Saving #{ platform }")
+      datasets.sort.each{|dataset|
+        path = Object::GEO.dataset_path(dataset)
+        next unless File.exists?(path + '_cross_platform.experiments')
+        experiments = File.open(path + '_cross_platform.experiments').collect{|l| l.chomp}
+        orders = File.open(path + '_cross_platform.orders').collect{|l| values = l.chomp.split(/\t/).collect{|v| v == "NA" ? nil : v.to_i };}
+
+        data = {}
+        codes.each_with_index{|code,i|
+          data[code.to_sym] = orders[i]
+        }
+
+        case 
+        when codes.length < 65535
+          type = "SMALLINT UNSIGNED"
+        when codes.length < 16777215
+          type = "MEDIUMIN UNSIGNED"
+        else
+          type = "INT UNSIGNED"
+        end
+
+
+        DBcache.save(dataset + '_cross_platform_experiments', experiments)
+        DBcache.save(dataset + '_cross_platform', data, [type] * orders.first.length)
+      }
+    end
+
+    def self.positions(platform, genes)
+      return [{},[]] if genes.empty?
+      genes = genes.collect{|gene| gene.downcase.strip}
+
+      datasets = Object::GEO.platform_datasets(platform).sort
+      platform_entries = platform_entries(platform).to_f
+
+      data = {}
+      matched = nil
+
+      datasets.each{|dataset|
+        dataset += '_cross_platform' if Object::GEO::is_cross_platform?(platform)
+        gene_positions = DBcache.load(dataset, genes)
+        matched ||= gene_positions.keys
+        
+        experiments = DBcache.load(dataset + '_experiments').sort{|a,b| 
+          a[0].to_i <=> b[0].to_i
+        }.collect{|p| 
+          Object::GEO::clean(dataset) + ": " + p[1].first
+        }
+
+        scale = (0..experiments.length - 1).collect{|i| 
+          rows = DBcache.num_rows(dataset, "C#{i}"); 
+          if rows > 0 
+            platform_entries / rows 
+          else 
+            nil 
+          end 
+        }
+        
+        gene_x_experiment = gene_positions.values
+
+        experiment_x_gene = gene_x_experiment.transpose
+
+        experiments.each_with_index{|experiment, i|
+          next if scale[i].nil? || experiment_x_gene[i].nil?
+          values = experiment_x_gene[i].collect{|v| v.nil? ? nil : (v.to_f * scale[i]).to_i}
+          data[experiment] = values
+        }
+      }
+
+      [data, matched]
+    end
+
+    def self.platform_entries(platform)
+      DBcache.num_rows(platform)
+    end
+  end
+
+end
+
+if __FILE__ == $0
+  #CustomDS::datasets('sgd').each{|d| MADB::CustomDS::save(d)}
+
+  require 'pp'
+  pp MADB::GEO::positions('GPL91_cross_platform', %w(2778))[0].select{|k,v| k =~ /GDS989/}.sort
+  #p MADB::CustomDS::positions("HaploidData",%w( YMR261c YDL140c YIL122w YPL093w YHR211w YDL142c YHR106w YOR103c YDR233c YLR181c yomeman))
+  #p MADB::CustomDS::positions("HaploidData_cross_platform",%w( S000002685 S000001149 S000003068 S000003153 S000003355 S000000127 S000004444 S000004875 S000001702 S000005843 S000000862))
+end

data/lib/MARQ/annotations.rb

@@ -0,0 +1,740 @@
+require 'inline'
+require 'net/http'
+require 'uri'
+require 'MARQ'
+require 'rbbt/bow/dictionary'
+require 'MARQ/fdr'
+
+module Annotations
+  class << self
+    inline do |builder|
+      builder.c_raw <<-EOC
+
+/**
+ * Compute log(k!)
+ * @param k The value k.
+ * @return The result.
+ */
+double lFactorial(double k)
+{
+	double r = 0;
+	int i;
+	for(i=2 ; i<=(int)k ; i++)
+	{
+		r = r + (double)(log((double)i));
+	}
+	return r;
+}
+
+
+
+/**
+ * Compute the log(binom(n,k))
+ * @param n The number of possible items.
+ * @param k The number of selected items.
+ * @return The result.
+ */
+double lBinom(double n, double k)
+{
+	long i;
+	double r = 0;
+	
+	if(n > n-k){
+		k = n-k;
+	}
+
+	for(i = (long)n ; i> (n-k) ; i--)
+	{
+		r = r + log((double)i);
+	}
+		
+	r = r - lFactorial(k);
+	
+	return r;
+}
+  EOC
+  
+      builder.c <<-EOC
+/**
+*  * Compute the Hypergeometric accumulated value.
+*  * @param total => total size
+*  * @param support => total support
+*  * @param list => selected list size,
+*  * @param found => support
+*  * @return The result
+*  */
+double hypergeometric(double total, double support, double list, double found)
+{
+	double other = total - support;
+
+	double top = list;
+	if(support < list){
+		top = support;
+	}
+
+	double log_n_choose_k = lBinom(total,list);
+
+	double lfoo = lBinom(support,top) + lBinom(other, list-top);
+	
+	double sum = 0;
+  int i;
+	for (i = (int)top; i >= found; i-- )
+	{
+		sum = sum + exp(lfoo - log_n_choose_k);
+		if ( i > found)
+		{
+			lfoo = lfoo + log(i / (support - i+1)) +  log( (other - list + i) / (list-i+1)  );
+		}
+	}
+	return sum;
+}
+      EOC
+    end
+  end
+
+  def self.exp2gds(experiment)
+    experiment =~ /(.*?):/
+    $1
+  end
+
+  def self.compare(a,b)
+    case 
+    when a[:pvalue] < b[:pvalue]
+      -1 
+    when a[:pvalue] > b[:pvalue]
+      1 
+    when a[:pvalue] == b[:pvalue]
+      b[:score].abs <=> a[:score].abs
+    end
+  end
+
+  RANK_SIZE_BINS = %w(1 2 3 4 5 7 10 15 20 30 40 50 65 80 100 125 150 175 200 250 300 350 400 450 500 600 700 800 900 1000 1500 2000 2500 3000)
+
+  def self.enrichment_rank(annotations, ranks, options = {})
+    dict_options = {
+      :dict_options => {:low => 0, :hi => 0.5, :limit => 100000}
+    }.merge(options)[:dict_options]
+    positions = {}
+    found_datasets = []
+
+    dict = Dictionary::TF_IDF.new
+    ranks.each_with_index{|experiment, rank|
+      info = annotations[experiment]
+
+      dataset_terms   = info[:dataset]
+      signature_terms = info[:signature]
+      
+      dataset = exp2gds experiment
+
+      terms = signature_terms
+      terms += dataset_terms
+
+      term_count = {}
+      terms.each{|term|
+        term_count[term] ||= 0
+        term_count[term] += 1
+      }
+      dict.add(term_count)
+    }
+ 
+    best = dict.best(dict_options).keys
+
+    found_datasets = []
+    ranks.each_with_index{|experiment, rank|
+      info = annotations[experiment]
+
+      dataset_terms   = info[:dataset]
+      signature_terms = info[:signature]
+
+      dataset = exp2gds experiment
+
+      terms = signature_terms
+      
+      if ! found_datasets.include? dataset
+        terms += dataset_terms
+        found_datasets << dataset
+      end
+
+      terms.uniq.each{|term|
+        next if not best.include? term
+        positions[term] ||= []
+        positions[term] <<  rank
+      }
+    }
+ 
+    scores = []
+
+
+    sizes = {}
+    RANK_SIZE_BINS.each{|size| sizes[size.to_i] = []}
+
+
+    # For each term compute the rank score. Also, place it in the closest size
+    # bin for the permutations.
+    best.each_with_index{|term, pos|
+      if positions[term]
+        list = positions[term]
+
+        # place it on the size bin 
+        found = false
+        sizes.keys.sort.each_with_index{|size,i|
+          next if found
+          if list.length < size
+            found = true
+            sizes[sizes.keys.sort[i-1]] << pos
+          end
+        }
+        sizes[sizes.keys.sort.last] << pos if !found
+        
+        scores << Score::score(list, ranks.length, 0)[:score]
+      else # it has no score
+        scores << nil
+      end
+    }
+
+    info = {}
+
+    # Go through all the size bins, run the permutations and assign the pvalues
+    # to all terms in the bin.
+    sizes.keys.each{|size|
+      next if size == 1
+      next if sizes[size].empty?
+
+      # This are the actual scores for the terms in the bin
+      sub_list_scores = sizes[size].collect{|pos| scores[pos] || 0}
+
+      # Compute the pvalues for all the terms in the bin. The size of the
+      # permutation list is that of the bin
+      pvalues = Score::pvalues(sub_list_scores, size, 0, ranks.length)
+
+      # Save the information from the terms, score, hits, and pvalues.
+      sizes[size].zip(pvalues).each{|p|
+        pos = p[0]
+        pvalue = p[1]
+        score = scores[pos]
+        next if score < 0
+
+        term = best[pos]
+        hits = positions[term].nil? ? 0 : positions[term].length 
+
+        info[term] = {:score => score, :hits => hits, :pvalue => pvalue}
+      }
+    }
+
+    info
+  end
+
+  def self.enrichment_hypergeometric(annotations, relevant, options)
+    dict_options = {
+      :dict_options => {:low => 0, :hi => 0.5, :limit => 100000}
+    }.merge(options)[:dict_options]
+    positions = {}
+    found_datasets = []
+
+    dict = Dictionary::TF_IDF.new
+    ranks.each_with_index{|experiment, rank|
+      info = annotations[experiment]
+
+      dataset_terms   = info[:dataset]
+      signature_terms = info[:signature]
+      
+      dataset = exp2gds experiment
+
+      terms = signature_terms
+      terms += dataset_terms
+
+      term_count = {}
+      terms.each{|term|
+        term_count[term] ||= 0
+        term_count[term] += 1
+      }
+      dict.add(term_count)
+    }
+ 
+    best = dict.best(dict_options).keys
+
+    terms = {}
+    found_datasets = []
+    annotations.each{|experiment, info|
+      dataset_terms   = info[:dataset]
+      signature_terms = info[:signature]
+
+      dataset = exp2gds experiment
+
+      signature_terms.each{|term|
+        next if ! best.include? term
+        terms[term] ||= {:relevant => 0, :total => 0}
+        terms[term][:total]    += 1 
+        terms[term][:relevant] += 1 if relevant.include? experiment
+      }
+      
+      next if found_datasets.include? dataset
+      found_datasets << dataset
+
+      dataset_terms.each{|term|
+        next if ! best.include? term
+        terms[term] ||= {:relevant => 0, :total => 0}
+        terms[term][:total]    += 1 
+        terms[term][:relevant] += 1 if relevant.include? experiment
+      }
+    }
+ 
+
+    total   = annotations.keys.length
+    list    = relevant.length
+
+    terms.each{|term, info|
+      info[:pvalue] = Annotations.hypergeometric(total,info[:total],list, info[:relevant])
+    }
+
+    terms
+  end
+
+
+
+  def self.annotations(scores, type, pvalue = 0.05, algorithm = :rank) 
+    annot = {}
+    relevant = []
+
+    dict_options = {}
+    if type == "Words"
+      dict_options = {:low => 0, :hi => 0.05, :limit => 100000}
+    else
+      dict_options = {:low => 0, :hi => 0.5, :limit => 100000}
+    end
+
+    case
+    when type =~ /^(.*)_direct$/
+      side = :direct
+      type = $1
+    when type =~ /^(.*)_inverse$/
+      side = :inverse
+      type = $1
+    end
+
+
+    terms_cache = {}
+    scores.each{|experiment, info|
+      dataset = experiment.match(/^(.*?): /)[1]
+      name = $'.strip 
+      case
+      when side.nil?
+        term_file = File.join(MARQ.datadir, MARQ.platform_type(dataset).to_s , 'annotations',type, dataset)
+      when side == :direct && info[:score] > 0 || side == :inverse && info[:score] < 0
+        term_file = File.join(MARQ.datadir, MARQ.platform_type(dataset).to_s , 'annotations',type + '_up', dataset)
+      else
+        term_file = File.join(MARQ.datadir, MARQ.platform_type(dataset).to_s , 'annotations',type + '_down', dataset)
+      end
+
+      if File.exist? term_file
+        terms_cache[term_file] ||= YAML::load(File.open(term_file))
+        terms = terms_cache[term_file] 
+        annot[experiment] = {:dataset => (terms[:dataset] || []), :signature => (terms[name] || [])}
+      else
+        annot[experiment] = {:dataset =>  [], :signature => []}
+      end
+
+      relevant << experiment if info[:pvalue] <= pvalue
+    }
+
+    if algorithm == :rank
+      ranks = scores.sort{|a,b| compare(a[1],b[1]) }.collect{|p| p[0]}
+      terms = enrichment_rank(annot, ranks, dict_options)
+    else
+      terms = enrichment_hypergeometric(annot, relevant, dict_options)
+    end
+
+    merged_annotations = {}
+    annot.each{|key, info|
+      merged_annotations[key] = info[:dataset] + info[:signature]
+    }
+    [merged_annotations, terms]
+  end
+
+  module GO
+    module Genecodis
+      ORGS = {
+        'sgd' => 'Sc' ,
+        'rgd' => 'Rn' ,
+        'mgi' => 'Mm' ,
+        'pombe' => 'Sp' ,
+        'cgd' => 'Ca' ,
+        'human' => 'Hs' ,
+        'tair' => 'At' ,
+        'worm' => 'Ce' ,
+      }
+
+      FIELDS = %w(Id Items S TS Hyp Hyp_c Genes).collect{|f| f.downcase.to_sym}
+
+      module WS
+        def self.driver
+          require 'soap/wsdlDriver'
+          wsdl_url = File.join('http://genecodis.dacya.ucm.es/static/wsdl/genecodisWS.wsdl')
+          driver = SOAP::WSDLDriverFactory.new(wsdl_url).create_rpc_driver
+          driver
+        end
+
+        def self.analysis(org, list)
+          puts "GO for #{ org } #{list.length} genes"
+
+          gc_org = ORGS[org.to_s]
+          return [] if gc_org.nil?
+
+          job_id = driver.analyze(gc_org,2,0,-1,3,list,%w(GO_Biological_Process ),[])
+
+
+          while (stat = driver.status(job_id)) == 1
+            sleep 1
+          end
+
+          if stat < 0
+            return []
+          else
+            lines =  driver.results(job_id).collect{|l| l.chomp}
+            lines.shift
+            lines.collect{|l| Hash[*FIELDS.zip(l.chomp.split(/\t/)).flatten]}
+          end
+        rescue
+          puts $!.message
+          puts $!.backtrace
+        end
+      end
+
+      module Local
+
+        def self.analysis(org,list, slim = false)
+          require 'genecodis'
+
+          gc_org = ORGS[org.to_s]
+          if slim
+            groups = ['GOSlim_Process']
+          else
+            groups = ['GO_Biological_Process']
+          end
+
+          job_id = Object::Genecodis.analyze(gc_org,2,0,-1,3,list,groups,nil)
+          return [] if job_id.nil?
+
+          while (stat = Object::Genecodis.status(job_id)) == 1
+            sleep 0.5
+          end
+
+          if stat < 0
+            return []
+          else
+            res =  Object::Genecodis.results(job_id)
+            return [] if res.nil?
+            lines = res.collect{|l| l.chomp}
+            lines.shift
+            lines.collect{|l| Hash[*FIELDS.zip(l.chomp.split(/\t/)).flatten]}
+          end
+        rescue
+          puts $!.message
+          puts $!.backtrace
+        end
+      end
+    end
+
+    def self.get_genes_nth(dataset, num_genes)
+      path = MARQ.dataset_path(dataset)
+
+      experiments = File.open(path + '.experiments').collect{|l| l.chomp.strip}
+      genes       = File.open(path + '.codes').collect{|l| l.chomp.strip}
+      total_genes = genes.length
+
+      genes_up = {}
+      genes_down = {}
+      experiments.each{|exp| genes_up[exp] = []; genes_down[exp] = []}
+
+      File.open(path + '.orders').each_with_index{|l, i|
+        values = l.chomp.split(/\t/)
+        experiments.zip(values).each{|p|
+          name = p.first
+          value = p.last
+          next if p.last == "NA"
+          genes_up[name]   <<  genes[i] if value.to_i < num_genes
+          genes_down[name] <<  genes[i] if value.to_i > total_genes - num_genes
+        }
+      }
+
+      {:up => genes_up, :down => genes_down}
+    end
+
+    def self.get_genes(dataset, options = {})
+      fdr, cut_off, folds, do_folds, nth_genes = {
+        :fdr => false,
+        :cut_off => 0.05,
+        :folds => 2.5,
+        :do_folds => true,
+        :nth_genes => 0,
+      }.merge(options).values_at(:fdr, :cut_off, :folds, :do_folds, :nth_genes)
+
+      if nth_genes > 0
+        return get_genes_nth(dataset, nth_genes)
+      end
+      
+
+      path = MARQ.dataset_path(dataset)
+
+      experiments = File.open(path + '.experiments').collect{|l| l.chomp.strip}
+      genes       = File.open(path + '.codes').collect{|l| l.chomp.strip}
+
+
+      experiments_ts   = experiments.select{|exp| exp !~ /\[ratio\]/}
+      experiments_fold = experiments.select{|exp| exp =~ /\[ratio\]/}
+
+      experiments_fold = [] if ! do_folds
+
+      values_up = {}
+      values_down = {}
+      experiments.each{|exp| values_up[exp] = []; values_down[exp] = []}
+
+      File.open(path + '.pvalues').each_with_index{|l, i|
+        values = l.chomp.split(/\t/)
+        experiments_ts.zip(values).each{|p|
+          name = p.first
+          value = p.last == "NA" ? 1.0 : p.last.to_f
+          values_up[name]     << (value > 0 ? value : 1.0)
+          values_down[name]   << (value < 0 ? - value : 1.0)
+        }
+      }
+
+      File.open(path + '.logratios').each_with_index{|l, i|
+        values = l.chomp.split(/\t/)
+        experiments.zip(values).each{|p|
+          name = p.first
+          next unless experiments_fold.include? name
+          value = p.last == "NA" ? 0 : p.last.to_f
+          values_up[name]     << (p.last > 0 ? value : 0)
+          values_down[name]   << (p.last < 0 ? - value : 0)
+        }
+      }
+     
+      genes_up = {}
+      genes_down = {}
+
+      threshold = cut_off
+      values_up.each{|experiment, values|
+        genes_up[experiment] = []
+        if experiments_ts.include? experiment
+          if fdr
+            threshold = FDR.step_up(values.sort, cut_off)
+            next if threshold == 0.0
+          end
+          values.each_with_index{|value, i| genes_up[experiment] << genes[i] if value < threshold}
+        elsif experiment_fold.include? experiment
+          values.each_with_index{|value, i| genes_up[experiment] << genes[i] if value < folds}
+        end
+      }
+      values_down.each{|experiment, values|
+        genes_down[experiment] = []
+        if experiments_ts.include? experiment
+          if fdr
+            threshold = FDR.step_up(values.sort, cut_off)
+            next if threshold == 0.0
+          end
+          values.each_with_index{|value, i| genes_down[experiment] << genes[i] if value < threshold}
+        elsif experiment_fold.include? experiment
+          values.each_with_index{|value, i| genes_down[experiment] << genes[i] if value < folds}
+        end
+      }
+
+
+      {:up => genes_up, :down => genes_down}
+    end
+
+    def self.get_genes_old(dataset, cut_off = 0.1, fdr = false)
+
+      path = MARQ.dataset_path(dataset)
+
+      experiments = File.open(path + '.experiments').collect{|l| l.chomp.strip}.select{|name| !name.match(/\[ratio\]/)}
+      genes = File.open(path + '.codes').collect{|l| l.chomp.strip}
+
+
+      values_up = {}
+      values_down = {}
+      experiments.each{|exp| values_up[exp] = []; values_down[exp] = []}
+
+
+      if File.exist?(path + '.pvalues')
+        File.open(path + '.pvalues').each_with_index{|l, i|
+          values = l.chomp.split(/\t/)
+          experiments.zip(values).each{|p|
+            value = p.last == "NA" ? 1.0 : p.last.to_f
+            values_up[p.first]     << (value > 0 ? value : 1.0)
+            values_down[p.first]   << (value < 0 ? - value : 1.0)
+          }
+        }
+      end
+
+      genes_up = {}
+      genes_down = {}
+
+      threshold = cut_off
+      values_up.each{|experiment, values|
+        genes_up[experiment] = []
+        if fdr
+          threshold = FDR.step_up(values.sort, cut_off)
+          next if threshold == 0.0
+        end
+        values.each_with_index{|value, i| genes_up[experiment] << genes[i] if value < threshold}
+      }
+      values_down.each{|experiment, values|
+        genes_down[experiment] = []
+        if fdr
+          threshold = FDR.step_up(values.sort, cut_off)
+          next if threshold == 0.0
+        end
+        values.each_with_index{|value, i| genes_down[experiment] << genes[i] if value < threshold}
+      }
+
+
+      {:up => genes_up, :down => genes_down}
+    end
+  end
+  module UMLS
+    SEMANTIC_TYPES="T020,T100,T116,T123,T023,T118,T043,T049,T103,T200,T060,T047,T203,T126,T050,T131,T125,T129,T037,T197,T191,T114,T110,T167,T024"
+
+    def self.OBA(text)
+
+      res = Net::HTTP.post_form(URI.parse('http://rest.bioontology.org/obs_hibernate/annotator'),
+        {
+          'longestOnly'=> true, 
+          'wholeWordOnly'=> true,
+          'withDefaultStopWords' => true,
+          'scored' => true,
+          'ontologiesToExpand' => 'null', 
+          'ontologiesToKeepInResult' => "",
+          'levelMax' => 0,
+          'levelMin' => 0,
+          'textToAnnotate' => text.gsub(/\s/,' '),
+          'semanticTypes' => SEMANTIC_TYPES,
+          'mappingTypes' => 'null',
+          'format' => 'tabDelimited'
+      })
+
+      res.body.collect{|l| l.split(/\t/)}.select{|v| v[0].to_i > 0}.collect{|v| v[2].sub(/\W*NOS/,'').downcase}.select{|w| w !~ /^\d+$/}.sort.uniq
+    end
+
+  end
+
+
+end
+
+
+
+if __FILE__ == $0
+  require 'pp'
+
+  
+  
+  exit
+  #Annotations::GO::Genecodis::Local.init
+ 
+  #genes = Annotations::GO::get_genes('GDS1916')
+  #genes[:up].each{|exp, genes|
+  #  puts exp
+  #  p Annotations::GO::Genecodis::Local.analysis('mgi', genes)
+  #}
+  #exit
+  #res = Annotations::GO::get_genes('GDS948')
+  #res[:down].each{|exp, values| puts "#{ exp }\t#{ values.length }"}
+
+  texts = []
+  texts << <<-EOT
+
+  Analysis of femurs and tibias of growth hormone (GH) deficient animals at 6
+  and 24 hours following treatment with 4 mg/kg body weight GH. Results provide
+  insight into the insulin-like growth factor-I dependent and independent
+  pathways that mediate the action of GH in bone.
+  
+  EOT
+  texts << <<-EOT
+  
+  Comparison of total transcription profiles for temperature-sensitive TOR2
+  mutant strain SH121 to its isogenic wild type counterpart SH100. Results
+  indicate that TOR2 inactivation leads to enhanced transcription of
+  Gcn4-controlled target genes.
+  
+
+  EOT
+  texts << <<-EOT
+
+  Analysis of tissue specimens representing benign nevus, atypical nevus,
+  melanoma in situ, vertical growth phase (VGP) melanoma, and metastatic growth
+  phase (MGP) melanoma. Results identify expression signatures that distinguish
+  benign and atypical nevi and melanomas in situ from VGPs and MGPs.
+  
+  
+  EOT
+  texts << <<-EOT
+
+  Analysis of estrogen receptor (ER)-positive MCF7 breast cancer cells up to 48
+  hours following treatment with estradiol (E2). ERs facilitate the
+  transcriptional effects of hormones. These results, together with ChIP-PET
+  results, suggest potential correlations between ER binding and gene
+  regulation.
+
+  EOT
+  texts << <<-EOT
+
+  
+  Analysis of anaerobic chemostat cultures of Saccharomyces cerevisae exposed
+  to one of several weak organic acids. Weak organic acids are used as
+  preservatives in food and beverages. Yeasts are able to proliferate at the
+  maximum legal dosage of such preservatives.
+
+  EOT
+  texts << <<-EOT
+
+  Zucker diabetic fatty model of type 2 diabetes: various insulin-sensitive
+  tissues Analysis of adipose, skeletal muscle, and liver tissues of Zucker
+  diabetic fatty animals at pre-diabetic and diabetic stages. ZDF animals have
+  a mutated leptin receptor. Results provide insight into the molecular
+  mechanisms responsible for insulin resistance and progression to type 2
+  diabetes.
+
+  EOT
+  texts << <<-EOT
+
+  Strain differences in copper sulfate Each strain was grown overnight then
+  diluted in fresh rich media. After three hours strains were again rediluted
+  into either rich media or rich media supplemented with copper sulfate. After
+  another three hours cultures were sampled, cells lysed and flash frozen using
+  liquid nitrogen. RNA was extrated using hot phenol-chloroform, reverse
+  transcripbed using amino-allyle dUTP and labelled with either Cy3 or Cy5
+  flourescent dye. Each hybridization is of a single sample compared to a
+  reference pool contructed from all the strains with the same treatment.
+    Labelled probe were hybridized to DNA microarrays spotted with 6144 70 bp
+  oligonucleotides obtained from Qiagen-Operon. After an overnight
+  hybridization, microarrays were scanned using a GenePix 4000A scanner and
+  spot intensities extracted using GenePix 4.0 software. Bad spots were flagged
+  based on the image.
+
+
+  EOT
+  texts << <<-EOT
+
+  MyD88-deficient macrophage response to lipopolysaccharide and E. coli (dye swap)
+  Analysis of MyD88 null mutant macrophages treated with LPS or live E. coli.
+  MyD88 transduces cell signaling events downstream of Toll-like receptors, a
+  key component of host defense. Results suggest most of the host response to
+  endotoxin or live bacteria is actually regulated independently of MyD88.
+
+  
+  EOT
+  texts.reverse.each{|text|
+    puts "\n\n--------------\n"
+    puts "Text: "
+    puts "\n" + text.strip + "\n\n\n"
+    puts "Annotations: "
+    puts
+    puts Annotations::UMLS::OBA(text).join(", ")
+  }
+
+#
+
+  #puts  Annotations.hypergeometric(2000,100,100,2)
+  #p Annotations::GO::annotate(MARQ.platform_organism('GDS1365'),genes[:up].collect.first.last[1..100])
+end
+