rbbt-marq 1.1.0 → 2.0.0

data/install_scripts/rake_includes.rb

@@ -0,0 +1,236 @@
+$expr_threshold ||= (ENV['threshold'] || 0.05).to_f
+$folds          ||= (ENV['folds'] || 2.5).to_f
+$nth_genes      ||= (ENV['nth_genes'] || 100).to_i
+
+$force       = [$force, ENV['force'], false].compact.first.to_s == 'true'  
+$tranlations = [$tranlations, ENV['translations'], false].compact.first.to_s == 'true'  
+$series      = [$series, ENV['series'], true].compact.first.to_s == 'true' 
+$update_db   = [$update_db, ENV['update_db'], false].compact.first.to_s == 'true'  
+$skip_db     = [$skip_db, ENV['skip_db'], false].compact.first.to_s == 'true'  
+$fdr         = [$fdr, ENV['fdr'], true].compact.first.to_s == 'true'  
+$do_folds    = [$do_folds, ENV['do_folds'], true].compact.first.to_s == 'true'  
+
+
+$changes = false
+module GEO::Process::R
+  class << self
+    alias_method :GDS_old, :GDS
+    def GDS(*args)
+      $changes = true
+      GDS_old(*args)
+    end
+  
+    alias_method :GSE_old, :GSE
+    def GSE(*args)
+      $changes = true
+      GSE_old(*args)
+    end
+  end
+end
+
+desc "Analyze datasets"
+task 'data' do 
+
+  platforms_to_save = []
+
+  platforms = process_list
+  platforms.each{|platform, datasets|
+
+    begin
+      # Prepare the platform
+      MARQ::Platform.process(platform)
+    rescue
+      puts "Error processing platform #{platform}"
+      puts $!.message
+      puts $!.backtrace.join("\n")
+      next
+    end
+    
+    next if $tranlations
+
+    $changes = false
+    # Process all datasets
+    datasets.each{|dataset|
+      begin
+        next unless $force || MARQ::Dataset.path(dataset).nil?
+        MARQ::Dataset.process(dataset, platform)
+      rescue
+        puts "Error processing dataset #{ dataset }"
+        puts $!.message
+        puts $!.backtrace.join("\n")
+      end
+    }
+
+    # Mark the platform for saving in DB
+    platforms_to_save << platform if $changes || $update_db
+  }
+
+  platforms_to_save.each{|platform|
+    begin
+      puts "Saving #{platform}"
+      MADB.save_platform(platform) 
+    rescue
+      puts "Error saving platform #{ platform }"
+      puts $!.message
+      puts $!.backtrace.join("\n")
+    end
+  }
+end
+
+def annotations(name, cross_platform = false, &block)
+  platforms = process_list
+
+  platforms.each do |platform, datasets|
+    datasets.each do |dataset|
+      begin
+        next if File.exist?(File.join("annotations", name, dataset)) && ! $force
+        next if MARQ::Dataset.path(dataset).nil?
+
+        FileUtils.mkdir_p File.join("annotations", name)
+        filename = File.join("annotations", name, dataset)
+        dataset += '_cross_platform' if cross_platform && MARQ::Platform::has_cross_platform?(platform)
+        next if ! MARQ::Dataset.exists?(dataset)
+        terms = block.call(dataset)
+        Open.write(filename, terms.to_yaml)
+      rescue
+        puts "Error processing dataset #{ dataset }"
+        puts $!.message
+        puts $!.backtrace.join("\n")
+      end
+    end
+  end
+end
+
+
+task 'annotate_Words' do
+  require 'MARQ/annotations'
+  require 'rbbt/bow/bow'
+  annotations('Words') do |dataset|
+    terms = {}
+    description = Open.read(MARQ::Dataset.path(dataset) + '.description')
+    terms[:dataset] = [dataset] +  description.words.uniq
+    Open.read(MARQ::Dataset.path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = name.sub(/.*?: /,'').sub(/\[ratio\]/,'').words.uniq
+    }
+    terms
+  end
+end
+
+
+task 'annotate_UMLS' do
+  require 'MARQ/annotations'
+  require 'rbbt/util/misc'
+  annotations('UMLS') do |dataset|
+    terms = {}
+    description = Open.read(MARQ::Dataset.path(dataset) + '.description')
+    terms[:dataset] = Annotations::UMLS::OBA(description).uniq
+    Open.read(MARQ::Dataset.path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = Annotations::UMLS::OBA(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).uniq
+    }
+    terms
+  end
+end
+
+
+task 'annotate_Polysearch' do
+  require 'MARQ/annotations'
+  require 'rbbt/util/misc'
+  require 'rbbt/sources/polysearch'
+  annotations('Polysearch') do |dataset|
+    terms = {}
+    description = Open.read(MARQ::Dataset.path(dataset) + '.description')
+    terms[:dataset] = Polysearch::match(description).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
+    Open.read(MARQ::Dataset.path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = Polysearch::match(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
+    }
+    terms
+  end
+
+end
+
+def goterms(org, list, slim, threshold)
+  return [] if list.empty?
+  results = Annotations::Genes::Genecodis::Local.analysis(org, list, slim)
+  return [] if results.nil?
+  results.
+    select{|info| info[:s].to_i  > 2 }.
+    select{|info| info[:hyp_c].to_f < threshold }.
+    collect{|info| info[:items]}.collect{|id| GO::id2name(id)}
+end
+
+task 'annotate_GO' do
+  require 'MARQ/annotations'
+  require 'rbbt/sources/go'
+  options = { :cut_off => $expr_threshold, :fdr => $fdr, :folds => $folds, :do_folds => $do_folds, :nth_genes => $nth_genes}
+  annotations('GO_up', true) do |dataset|
+    org = MARQ::Dataset.organism(dataset)
+    genes = Annotations::Genes.get_genes(dataset, options)
+
+    up = {}
+    genes[:up] ||= []
+    genes[:up].collect{|experiment,list|
+      up[experiment] =  goterms(org, list, false, $expr_threshold)
+    }
+    up
+  end
+
+  annotations('GO_down', true) do |dataset|
+    org = MARQ::Dataset.organism(dataset)
+    genes = Annotations::Genes.get_genes(dataset, options)
+
+    down = {}
+    genes[:down] ||= []
+    genes[:down].collect{|experiment,list|
+      down[experiment] = goterms(org, list, false, $expr_threshold)
+    }
+    down
+  end 
+
+  annotations('GOSlim_up', true) do |dataset|
+    org = MARQ::Dataset.organism(dataset)
+    genes = Annotations::Genes.get_genes(dataset, options)
+
+    up = {}
+    genes[:up] ||= []
+    genes[:up].collect{|experiment,list|
+      up[experiment] = goterms(org, list, true, $expr_threshold)
+    }
+    up
+  end
+
+  annotations('GOSlim_down', true) do |dataset|
+    org = MARQ::Dataset.organism(dataset)
+    genes = Annotations::Genes.get_genes(dataset, options)
+
+    down = {}
+    genes[:down] ||= []
+    genes[:down].collect{|experiment,list|
+      down[experiment] =  goterms(org, list, true, $expr_threshold)
+    }
+    down
+  end
+end
+
+task 'annotate_SENT' do
+  require 'MARQ/annotations'
+  options = { :cut_off => $expr_threshold, :fdr => $fdr, :folds => $folds, :do_folds => $do_folds, :nth_genes => $nth_genes}
+  annotations('SENT') do |dataset|
+    org = MARQ::Dataset.organism(dataset)
+    genes = Annotations::Genes.get_genes(dataset, options)
+    terms = Annotations::Genes::SENT.terms(org, genes)
+    terms
+  end
+
+
+end
+
+task 'default' do
+  Rake::Task['data'].invoke
+  Rake::Task['annotate_Words'].invoke
+  Rake::Task['annotate_UMLS'].invoke
+  Rake::Task['annotate_Polysearch'].invoke
+  Rake::Task['annotate_GO'].invoke
+end

data/lib/MARQ/CustomDS.rb

@@ -5,9 +5,8 @@ require 'MARQ/ID'
 module CustomDS
   @@r = nil
 
-  def self.customdir
-    File.join(MARQ.datadir,'CustomDS')
-  end
+  DATA_DIR = File.join(MARQ.datadir,'CustomDS')
+
 
   def self.r
     require 'rsruby'
@@ -35,33 +34,12 @@ module CustomDS
     end
   end
 
-  def self.path(dataset)
-    files = Dir.glob(customdir + "/*/#{ dataset }.orders")
-    if files.length == 1
-      files.first.sub(/.orders/,'')
-    else
-      Dir.glob(customdir + "/*/#{ dataset }").first
-    end
-  end
-
   def self.organism(dataset)
-    path(dataset).match(/#{ customdir }\/(.*?)\//)[1]
-  end
-
-  def self.is_cross_platform?(dataset)
-    dataset.match(/_cross_platform/)
-  end
-
-  def self.clean(dataset)
-    dataset.sub(/_cross_platform/,'')
-  end
-  
-  def self.has_cross_platform?(dataset)
-    Dir.glob(path(clean(dataset)) + '_cross_platform.orders').any?
+    path(dataset).match(/#{ DATA_DIR }\/(.*?)\//)[1]
   end
 
   def self.datasets(org)
-    Dir.glob(File.join(customdir, org) + '/*.orders').collect{|f| clean(File.basename(f.sub(/.orders/,'')))}.uniq
+    Dir.glob(File.join(DATA_DIR, org) + '/*.orders').collect{|f| clean(File.basename(f.sub(/.orders/,'')))}.uniq
   end
 
   def self.process_matrix(prefix, org)
@@ -82,9 +60,62 @@ module CustomDS
   end
 
   def self.process(name)
-    puts "Processing #{ name }"
-    org = organism(name)
-    prefix = File.join(customdir, org, name)
+   end
+
+  def self.organisms
+    Dir.glob(File.join(DATA_DIR, '*')).
+      select {|path| File.directory? path}.
+      collect {|path| File.basename path}
+  end
+
+  def self.dataset_path(dataset)
+    organisms.each do |organism|
+      case
+      when File.exists?(File.join(DATA_DIR, organism, dataset + '.orders'))
+        return File.join(DATA_DIR, organism, dataset)
+      when File.exists?(File.join(DATA_DIR, organism, dataset + '.skip'))
+        return nil
+      end
+    end
+    return nil
+  end
+
+  def self.platform_path(platform)
+    dataset_path(platform)
+  end
+
+  def self.platform_datasets(platform)
+    MARQ::Dataset.clean(platform)
+  end
+
+  def self.platform_organism(platform)
+    path = platform_path(platform)
+    return nil if path.nil?
+    path.match(/#{DATA_DIR}\/(.*)\/#{ platform }$/)
+    return $1
+  end
+
+  def self.dataset_organism(dataset)
+    platform_organism(dataset)
+  end
+
+  def self.dataset_platform(dataset)
+    dataset
+  end
+
+  def self.organism_platforms(organism)
+    Dir.glob(File.join(DATA_DIR,organism,'*.orders')).
+      collect {|path| File.basename(path).sub(/\.orders$/,'').sub(/_cross_platform/,'')}.
+      uniq
+  end
+
+  def self.process_platform(platform)
+  end
+
+  def self.process_dataset(dataset, platform = nil)
+    puts "Processing #{ dataset }"
+    org = dataset_organism(dataset)
+    prefix = File.join(DATA_DIR, org, dataset)
 
     CustomDS::process_matrix(prefix, org)
   end
@@ -94,13 +125,13 @@ end
 
 if __FILE__ == $0
   p CustomDS::datasets('sgd')
-  p CustomDS::path('HaploidData')
-  p CustomDS::path('HaploidData_cross_platform')
+  p CustomDS::dataset_path('HaploidData')
+  p CustomDS::dataset_path('HaploidData_cross_platform')
 
   exit
 
   org = 'sgd'
-  process = Dir.glob(File.join(CustomDS::customdir, org) + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - CustomDS.datasets('sgd')
+  process = Dir.glob(File.join(CustomDS::DATA_DIR, org) + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - CustomDS.datasets('sgd')
   p process
   process.each{|d| CustomDS::process(d)}
 

data/lib/MARQ/GEO.rb

@@ -1,45 +1,56 @@
 require 'MARQ'
+require 'MARQ/main'
 require 'rbbt/sources/organism'
 
 # Work with GEO datasets
 module GEO
 
+  CACHE_DIR = File.join(MARQ.cachedir,'GEO')
+  FileUtils.mkdir_p CACHE_DIR unless File.exists? CACHE_DIR
+
+  DATA_DIR = File.join(MARQ.datadir, 'GEO')
+
   # Get information from Entrez
   module Remote
 
+    @@nice = 1
     def self.organism_platforms(org)
       name = Organism.name(org)
-      Open.read("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=gds&term=\"#{name}\"[Organism:exp]+AND+%22gpl%22[Filter]&retmax=10000").
+      Open.read("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=gds&term=\"#{name}\"[Organism:exp]+AND+%22gpl%22[Filter]&retmax=10000", :nice => @@nice).
         scan(/<Id>(\d+?)<\/Id>/).collect{|id| id.first}.collect{|id| "GPL#{id.sub(/^100*/,'')}"}
     end
 
     def self.platform_datasets(platform)
-      Open.read("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=gds&term=#{platform}[Accession]&retmax=2000").
+      Open.read("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=gds&term=#{platform}[Accession]&retmax=2000", :nice => @@nice).
       scan(/<Id>(\d+?)<\/Id>/).collect{|id| id.first}.select{|id| !id.match(/^(1|2)000/) }.collect{|id| "GDS#{id}"}
     end
 
     def self.dataset_platform(dataset)
       if dataset =~ /GSE/
-        Open.read("http://www.ncbi.nlm.nih.gov/projects/geo/query/acc.cgi?acc=#{dataset}").scan(/GPL\d+/).uniq.sort.join("_")
+        Open.read("http://www.ncbi.nlm.nih.gov/projects/geo/query/acc.cgi?acc=#{dataset}", :nice => @@nice).scan(/GPL\d+/).uniq.sort.join("_")
       else
-        Open.read("http://www.ncbi.nlm.nih.gov/sites/GDSbrowser?acc=#{dataset}").scan(/GPL\d+/).uniq.sort.join("_")
+        Open.read("http://www.ncbi.nlm.nih.gov/sites/GDSbrowser?acc=#{dataset}", :nice => @@nice).scan(/GPL\d+/).uniq.sort.join("_")
       end
     end
 
     def self.series_dataset?(gse)
-      Open.read("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=geo&term=#{gse}[Accession]&retmax=2000").
+      Open.read("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=geo&term=#{gse}[Accession]&retmax=2000", :nice => @@nice).
       match(/<Id>(\d+?)<\/Id>/) != nil
     end
 
+    def self.platform_organism(platform)
+      Open.read("http://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=#{platform}", :nice => @@nice).
+      match(%r#<td><a href="http://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi\?mode=Info&amp;id=\d+" onmouseout="onLinkOut\('HelpMessage' , geo_empty_help\)" onmouseover="onLinkOver\('HelpMessage' , geoaxema_organismus\)">(.*)</a></td>#)[1]
+    end
+
   end
 
-  CACHE_DIR = File.join(MARQ.cachedir,'GEO')
-  FileUtils.mkdir_p CACHE_DIR unless File.exists? CACHE_DIR
 
 
   # Parse information in .soft files
   module SOFT
 
+    @@nice = 1
     GEO_SOFT="http://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?targ=self&view=full&form=text&acc="
 
     # Download a soft file. Uses cache
@@ -49,7 +60,7 @@ module GEO
       if File.exist?( cache_file )
         File.open(cache_file).read
       else
-        content = Open.read(GEO_SOFT + item, :nocache => true)
+        content = Open.read(GEO_SOFT + item, :nocache => true, :nice => @@nice)
         raise "SOFT file error" if content !~ /!/
         fout = File.open(cache_file,'w')
         fout.write content
@@ -105,7 +116,7 @@ module GEO
       soft = get_soft(series)
 
       if match = soft.scan(/!Series_platform_id\s*=?\s*(.*)/)
-        platform = match.flatten.collect{|p| p.strip}
+        platform = match.flatten.collect{|p| p.strip}.join("_")
       else
         raise "No Platform information" 
       end
@@ -131,7 +142,7 @@ module GEO
       end
 
       {
-        :platform => platform.join("_"),
+        :platform => platform,
         :description =>description.strip,
         :title => title.strip,
         :samples => samples,
@@ -162,8 +173,8 @@ module GEO
     end
 
     def self.GPL(platform)
-      if !File.exist?(File.join(MARQ.datadir, 'GEO', 'platforms',"#{platform}.yaml"))  &&
-        !File.exist?(File.join(MARQ.datadir, 'GEO', 'platforms',"#{platform}.skip")) 
+      if !File.exist?(File.join(DATA_DIR, 'platforms',"#{platform}.yaml"))  &&
+         !File.exist?(File.join(DATA_DIR, 'platforms',"#{platform}.skip")) 
         begin
           if platform =~ /_/
             organism =  GPL(platform.match(/(.*?)_/)[1])[:organism]
@@ -230,17 +241,17 @@ module GEO
           info[:other_ID_field] = [other_pos + 1, other_name] if other_pos > 0
 
 
-          Open.write(File.join(MARQ.datadir, 'GEO', 'platforms',"#{platform}.yaml"), info.to_yaml)
+          Open.write(File.join(DATA_DIR, 'platforms',"#{platform}.yaml"), info.to_yaml)
         rescue Exception
           puts $!.message
           puts $!.backtrace
-          Open.write(File.join(MARQ.datadir, 'GEO', 'platforms',"#{platform}.skip"), $!.message)
+          Open.write(File.join(DATA_DIR, 'platforms',"#{platform}.skip"), $!.message)
         end
       end
 
       raise "Platform info for #{ platform } is not available and could not be automatically produced." if File.exist?(File.join(MARQ.datadir, 'GEO', 'platforms',"#{platform}.skip"))
 
-      YAML::load(File.open(File.join(MARQ.datadir, 'GEO', 'platforms',"#{platform}.yaml")))
+      YAML::load(File.open(File.join(DATA_DIR, 'platforms',"#{platform}.yaml")))
     end
 
 
@@ -365,6 +376,8 @@ module GEO
       if File.exist?(File.join(platform,'cross_platform'))
         puts "-- Translated to cross_platform format"
         R.GDS(dataset, prefix + '_cross_platform', field, File.join(platform_path, 'translations')) 
+      else
+        puts "No cross_platform probe ids for platform"
       end
     end
 
@@ -388,10 +401,11 @@ module GEO
       do_log = !info[:log2] if info[:log2]
       fields = info[:fields]
 
-      puts "Processing GSE #{ series }. Platform #{ platform }"
-
       platform_path = GEO::platform_path(platform)
+      return if platform_path.nil?
       prefix = File.join(platform_path, 'GSE', series.to_s)
+
+      puts "Processing GSE #{ series }. Platform #{ platform }"
       puts "-- Original"
       R.GSE(gsms, conditions, do_log, prefix, nil, fields, info[:title], info[:description])
 
@@ -433,6 +447,8 @@ module GEO
         fix_GSE_ids(File.join(platform_path, 'cross_platform'),prefix + '_cross_platform');
         FileUtils.cp(File.join(platform_path, 'cross_platform'),prefix + '_cross_platform.codes')
         FileUtils.rm(prefix + '.translations') if File.exist?(prefix + '.translations')
+      else
+        puts "No cross_platform probe ids for platform"
       end
       FileUtils.rm(prefix + '.swap') if File.exist?(prefix + '.swap')
     end
@@ -441,7 +457,7 @@ module GEO
     # system (called biomart for clarity)
     def self.GPL(platform)
       path = GEO::platform_path(platform)
-      return if File.exist? path
+      return if path.nil? || File.exist?(path)
 
       if platform =~ /_/
         FileUtils.mkdir(path)
@@ -522,81 +538,101 @@ module GEO
 
   end
 
+  def self.platforms
+    Dir.glob(File.join(DATA_DIR, "GPL*")).collect {|path| File.basename(path) }
+  end
 
 
-  #{{{ Local data store info
-
-  def self.clean(name)
-    name.sub(/_cross_platform/,'') if name
+  def self.dataset_type(dataset)
+    case
+    when dataset =~ /^GDS/
+      :GDS
+    when dataset =~ /^GSE/
+      :GSE
+    end
   end
 
-
   def self.platform_path(platform)
-    File.join(MARQ.datadir, "GEO/#{clean(platform)}")
+    path = File.join(DATA_DIR, platform)
+    path = nil unless File.exists? path
+    path
   end
 
+  def self.dataset_path(dataset, platform = nil)
+    if platform
+      platforms = [platform]
+    else
+      platforms = self.platforms
+    end
 
-  def self.is_cross_platform?(dataset)
-    dataset =~ /_cross_platform/
-  end
+    platforms.each do |platform|
+      platform_path = platform_path(platform)
+      next if platform_path.nil?
 
-  def self.has_cross_platform?(dataset = nil, platform = nil)
-    platform = clean(platform)
-    raise "Dataset #{ dataset } not found" if dataset && dataset_path(dataset, platform).nil? 
-    raise "Platform #{ platform } not found" if platform && platform_path(platform).nil? 
-    if dataset
-      File.exists?(dataset_path(dataset, platform) + "_cross_platform.orders")
-    else
-      Dir.glob(File.join(platform_path(platform), '*', '*_cross_platform.orders')).any?
+      prefix = File.join(platform_path, dataset_type(dataset).to_s, dataset)
+      case
+      when File.exists?(prefix + '.orders')
+        return File.join(platform_path, dataset_type(dataset).to_s, dataset)
+      when File.exists?(prefix + '.skip')
+        return nil
+      end
     end
+
+    return nil
   end
 
-  def self.dataset_path(dataset, platform = nil)
-    if platform
-      files = Dir.glob(File.join(platform_path(clean(platform)),"/*/#{ dataset }"))
+  def self.platform_datasets(platform)
+    cross_platform = MARQ::Platform.is_cross_platform? platform
+    
+    path = platform_path(MARQ::Platform.clean(platform))
+    return [] if path.nil?
+
+    datasets = Dir.glob(File.join(path, '*', '*.orders')).
+      collect {|path| File.basename(path).sub(/\.orders$/,'')}
+
+    if cross_platform
+      datasets.select  {|dataset| MARQ::Dataset.is_cross_platform? dataset }.
+               collect {|dataset| MARQ::Dataset.clean(dataset) }
     else
-      files = Dir.glob(File.join(MARQ.datadir, "GEO/GPL*/*/#{ dataset }.*"))
+      datasets.select  {|dataset| ! MARQ::Dataset.is_cross_platform? dataset }
     end
-    return nil if files.empty?
-    return files.first.match(/(.*)\./)[1]
   end
 
-  def self.organism_platforms(organism)
-    Dir.glob(File.join(MARQ.datadir, "GEO/GPL*")).collect{|f| 
-      File.basename(f)
-    }.select{|platform| 
-      SOFT.GPL(platform)[:organism] == organism &&
-        platform_datasets(platform).any?
-    }
+  def self.dataset_platform(dataset)
+    path = dataset_path(dataset)
+    return nil if path.nil?
+    path.match(/(GPL\d+)/)
+    return $1
   end
 
+  def self.platform_organism(platform)
+    GEO::SOFT.GPL(platform)[:organism]
+  end
 
-
-  def self.platform_datasets(platform)
-    Dir.glob(File.join(platform_path(platform),"*/*.orders")).collect{|f| File.basename(f).sub(/.orders$/,'')}.select{|d| !is_cross_platform?(d)}
+  def self.dataset_organism(dataset)
+    platform_organism(dataset_platform(dataset))
   end
 
-  def self.dataset_platform(dataset)
-    dataset_path(dataset).match(/(GPL\d+)/)
-    $1
+  def self.process_platform(platform)
+    GEO::Process.GPL(platform)
   end
 
-  def self.GDS_info(name)
-    begin
-      title, description = Open.read(dataset_path(name) + '.description').split(/\n--\n/).values_at(0,1)
-      {:title => title.strip, :description => description.strip}
-    rescue Exception
-      puts $!.message
-      {:title => "" , :description => "" }
+  def self.process_dataset(dataset, platform)
+    case dataset_type(dataset)
+    when :GDS
+      GEO::Process.GDS(dataset, platform)
+    when :GSE
+      info = YAML::load(File.open("series/#{ dataset }.yaml"))
+      FileUtils.rm("platforms/#{ info[:platform] }.skip") if File.exist?  "platforms/#{ info[:platform] }.skip"
+      GEO::Process.GSE(dataset, info)
     end
-
   end
 
-
 end
 
 
 if __FILE__ == $0
+  p GEO.dataset_path 'GDS2791_cross_platform', 'GPL96'
 
 end