rbbt-marq 2.0.0 → 2.1.0

data/bin/marq_config

@@ -65,7 +65,7 @@ end
 
 
 $USAGE =<<EOT
-#{__FILE__} <action> [<subaction>] [--force (true|false)] [--update_db (true|false)] [--platform <gpl>] [--series <gse>] [--organism <org>] [--port <number>] [--host <name>]
+#{__FILE__} <action> [<subaction>] [--force (true|false)] [--update_db (true|false)] [--platform <gpl>] [--dataset <gds>] [--series (true|false)] [--organism <org>] [--port <number>] [--host <name>]
   actions:
     * config:   Set paths for data, cache, and tmp directories
 
@@ -88,7 +88,7 @@ $USAGE =<<EOT
 EOT
 
 class Controller < SimpleConsole::Controller
-  params :string => {:t => :target, :p => :platform, :s => :series,  :o => :organism}, :boolean => {:d => :update_db, :f => :force}, :integer => {:p => :port}
+  params :string => {:d => :dataset, :t => :target, :p => :platform, :s => :series,  :o => :organism, :db => :update_db, :f => :force}, :integer => {:p => :port}
 
 
   def prepare
@@ -99,9 +99,10 @@ class Controller < SimpleConsole::Controller
   def install
     $platform  = params[:platform] unless params[:platform].nil?
     $series    = params[:series]   unless params[:series].nil?
+    $dataset    = params[:dataset]   unless params[:dataset].nil?
     $organism  = params[:organism] unless params[:organism].nil?
-    $update_db = params[:update_db] && params[:update_db] =~ /true|yes|y/i
-    $force     = params[:force]     && params[:force]     =~ /true|yes|y/i
+    $update_db = params[:update_db].match(/true|yes|y/i) != nil unless params[:update_db].nil?
+    $force     = params[:force].match(/true|yes|y/i) != nil unless params[:force].nil?
     @actions   = params[:id] || %w(GEO)
     @rake_action  = params[:target] || 'default' 
   end
@@ -186,25 +187,28 @@ class View < SimpleConsole::View
 
       @actions = [@actions] if @actions === String
       @actions.each{|action|
-        puts "Installing #{ action }"
+        puts "Prepare #{ action }"
         Rake::Task[action].invoke
       }
     end
   end
 
   def install
-
     require 'rake'
+
     @actions = [@actions] if @actions === String
 
-    @actions.each{|action|
+    @actions.each do |action|
+
       Thread.new{
-        puts "Updating #{action}. Target: #{@rake_action}"
-        FileUtils.cd File.join(MARQ.datadir, action)
+        puts "Install #{action}. Target: #{@rake_action}"
         load File.join(MARQ.datadir, action, 'Rakefile')
+
+        FileUtils.cd File.join(MARQ.datadir, action)
         Rake::Task[@rake_action].invoke
       }.join
-    }
+
+    end
   end
 
 

data/install_scripts/CustomDS/Rakefile

@@ -25,7 +25,7 @@ def process_list
     organisms.each do |organism|
       platforms = CustomDS.organism_platforms(organism)
       platforms.each do |platform|
-        list[platform] = platform
+        list[platform] = [platform]
       end
     end
   end

data/install_scripts/GEO/Rakefile

@@ -12,6 +12,7 @@ require File.join(File.dirname(File.dirname(__FILE__)), 'rake_includes')
 $platform       ||= ENV['platform']
 $organism       ||= [$organism, ENV['organism'], nil].compact.first 
 $dataset        ||= ENV['dataset']
+$series          = [$series, ENV['series'], true].compact.first.to_s == 'true' 
 
 # More global variables in rake_includes file
 
@@ -51,7 +52,7 @@ def process_list
         else
           organism =  GEO::Remote::platform_organism(platform)
         end
-        if organism.split(',').select{|org| organisms.include? Organism.name2org(org.strip)}.any?
+        if organism.split(',').select{|org| organisms.include?(Organism.name2org(org.strip)) || organisms.include?(org) }.any?
           list[platform] ||= []
           list[platform] << serie 
         end

data/install_scripts/GEO/series/GSE1814.yaml

@@ -0,0 +1,44 @@
+--- 
+:title: Transcriptional effects of the TOR2-controlled signaling function
+:description: |-
+  We analysed the transcriptional effects of the TOR2-controlled signaling function using a genome-wide microarray approach in yeast. In S. cerevisiae, TOR2 has two essential signaling functions. One, shared with TOR1, is required for translation initiation, transcription, and cell growth in response to the presence of nutrients. The second is unique to TOR2, and functions in cell-cycle-dependent actin polarization and possibly in transcription. A previous genetic screen for mutants defective in the TOR-shared and the TOR2-unique functions identified several TOR2 temperature-sensitive alleles. In this study, we compared total transcription profiles for strain SH121, which is specifically defective in the TOR2-unique function, and its isogenic wild type counterpart SH100.
+  Keywords = TOR
+  Keywords: time-course
+:arrays: 
+  GSM31661: 
+    time: 2H
+    treatment: SH121
+  GSM31662: 
+    time: 2H
+    treatment: SH121
+  GSM31663: 
+    time: 6H
+    treatment: SH121
+  GSM31664: 
+    time: 6H
+    treatment: SH121
+  GSM31653: 
+    time: 0H
+    treatment: SH100
+  GSM31654: 
+    time: 0H
+    treatment: SH100
+  GSM31655: 
+    time: 2H
+    treatment: SH100
+  GSM31656: 
+    time: 2H
+    treatment: SH100
+  GSM31657: 
+    time: 6H
+    treatment: SH100
+  GSM31658: 
+    time: 6H
+    treatment: SH100
+  GSM31659: 
+    time: 0H
+    treatment: SH121
+  GSM31660: 
+    time: 0H
+    treatment: SH121
+:platform: GPL90

data/install_scripts/GEO/series/GSE21.yaml

@@ -0,0 +1,44 @@
+--- 
+:title: snf/swi mutants of S. cerevisiae.
+:description: |-
+  The Saccharomyces cerevisiae Snf/Swi complex has been previously demonstrated to control transcription and chromatin structure of particular genes in vivo and to remodel nucleosomes in vitro. We have performed whole-genome expression analysis, using DNA microarrays, to study mutants deleted for a gene encoding one conserved (Snf2) or one unconserved (Swi1) Snf/Swi component. This analysis was performed on cells grown in both rich and minimal media. The microarray results, combined with Northern blot, computational, and genetic analyses, show that snf2Delta and swi1Delta mutations cause similar effects on mRNA levels, that Snf/Swi controls some genes differently in rich and minimal media, and that Snf/Swi control is exerted at the level of individual genes rather than over larger chromosomal domains. In addition, this work shows that Snf/Swi controls mRNA levels of MATalpha-specific genes, likely via controlling transcription of the regulators MATalpha1 and MCM1. Finally, we provide evidence that Snf/Swi acts both as an activator and as a repressor of transcription, and that neither mode of control is an indirect effect of the other.
+  This study is described in more detail in Sudarsanam P et al.(2000) Proc Natl Acad Sci U S A 97:3364-9
+  Keywords: other
+:arrays: 
+  GSM1011: 
+    mutante: snf2
+    medio: rich
+  GSM1012: 
+    mutante: swi1
+    medio: rich
+  GSM1013: 
+    mutante: snf2
+    medio: rich
+  GSM1014: 
+    mutante: snf2
+    medio: rich
+  GSM1015: 
+    mutante: swi1
+    medio: rich
+  GSM1004: 
+    mutante: swi1
+    medio: minimal
+  GSM1005: 
+    mutante: snf2
+    medio: minimal
+  GSM1006: 
+    mutante: swi1
+    medio: minimal
+  GSM1007: 
+    mutante: snf2
+    medio: minimal
+  GSM1008: 
+    mutante: swi1
+    medio: minimal
+  GSM1010: 
+    mutante: swi1
+    medio: rich
+  GSM1009: 
+    mutante: snf2
+    medio: minimal
+:platform: GPL57

data/install_scripts/GEO/series/GSE27.yaml

@@ -0,0 +1,22 @@
+--- 
+:title: Sporulation in yeast
+:description: |-
+  Diploid cells of budding yeast produce haploid cells through the developmental program of sporulation, which consists of meiosis and spore morphogenesis. DNA microarrays containing nearly every yeast gene were used to assay changes in gene expression during sporulation. At least seven distinct temporal patterns of induction were observed. The transcription factor Ndt80 appeared to be important for induction of a large group of genes at the end of meiotic prophase. Consensus sequences known or proposed to be responsible for temporal regulation could be identified solely from analysis of sequences of coordinately expressed genes. The temporal expression pattern provided clues to potential functions of hundreds of previously uncharacterized genes, some of which have vertebrate homologs that may function during gametogenesis.
+  This study is described in more detail in Chu S, et al. 1998. Science 282:699-705
+  Keywords: time-course
+:arrays: 
+  GSM1000: 
+    time: 0.5h
+  GSM995: 
+    time: 17h
+  GSM996: 
+    time: 19h
+  GSM998: 
+    time: 11h
+  GSM992: 
+    time: 0h
+  GSM993: 
+    time: 12h
+  GSM994: 
+    time: 15h
+:platform: GPL67

data/install_scripts/GEO/series/GSE5470.yaml

@@ -0,0 +1,19 @@
+--- 
+:arrays: 
+  GSM125326: 
+    condition: BY
+  GSM125327: 
+    condition: BY
+  GSM125328: 
+    condition: BY
+  GSM125330: 
+    condition: dbr1
+  GSM125329: 
+    condition: dbr1
+  GSM125331: 
+    condition: dbr1
+:description: |-
+  Introns in pre-mRNAs must be spliced out prior to their translation.  During splicing, introns are removed in the form of a lariat, in which the 5' end is linked to the 2' hydroxyl of an internal adenosine.  Lariat degradation is initiated by an 2'-5' phosphodiester-specific RNA endonuclease which debranches these lariat RNAs to linear form.  Deletion of the debranching enzyme is yeast results in the accumulation of lariat introns.  We used this accumulation to identify spliced lariat introns on a genome-wide scale in S. cerevisiae using tiling microarrays.
+  Keywords: two sample comparison, 3 biological replicates
+:title: Expression data from BY4743 and dbr1 yeast
+:platform: GPL4065

data/install_scripts/rake_includes.rb

@@ -1,10 +1,11 @@
+require 'progress-monitor'
+
 $expr_threshold ||= (ENV['threshold'] || 0.05).to_f
 $folds          ||= (ENV['folds'] || 2.5).to_f
 $nth_genes      ||= (ENV['nth_genes'] || 100).to_i
 
 $force       = [$force, ENV['force'], false].compact.first.to_s == 'true'  
 $tranlations = [$tranlations, ENV['translations'], false].compact.first.to_s == 'true'  
-$series      = [$series, ENV['series'], true].compact.first.to_s == 'true' 
 $update_db   = [$update_db, ENV['update_db'], false].compact.first.to_s == 'true'  
 $skip_db     = [$skip_db, ENV['skip_db'], false].compact.first.to_s == 'true'  
 $fdr         = [$fdr, ENV['fdr'], true].compact.first.to_s == 'true'  
@@ -28,12 +29,23 @@ module GEO::Process::R
   end
 end
 
+module CustomDS
+  class << self
+    alias_method :process_dataset_old, :process_dataset
+    def process_dataset(*args)
+      $changes = true
+      process_dataset_old(*args)
+    end
+  end
+end
+
 desc "Analyze datasets"
 task 'data' do 
-
   platforms_to_save = []
 
   platforms = process_list
+
+  Progress.monitor("Processing #{platforms.keys.length} platforms") if platforms.keys.length > 1
   platforms.each{|platform, datasets|
 
     begin
@@ -50,10 +62,15 @@ task 'data' do
 
     $changes = false
     # Process all datasets
+    
+    Progress.monitor("Processing #{datasets.length} datasets") if datasets.length > 1
     datasets.each{|dataset|
       begin
-        next unless $force || MARQ::Dataset.path(dataset).nil?
+        already_processed = MARQ::Dataset.exists?(dataset) || MARQ::Dataset.broken?(dataset)
+        next if already_processed && ! $force
+
         MARQ::Dataset.process(dataset, platform)
+        MARQ::Dataset.process(MARQ::Name.cross_platform(dataset), platform) if MARQ::Platform.has_cross_platform?(platform)
       rescue
         puts "Error processing dataset #{ dataset }"
         puts $!.message
@@ -65,9 +82,9 @@ task 'data' do
     platforms_to_save << platform if $changes || $update_db
   }
 
+  Progress.monitor("Saving #{platforms_to_save.length} platforms in DB") if platforms_to_save.length > 1
   platforms_to_save.each{|platform|
     begin
-      puts "Saving #{platform}"
       MADB.save_platform(platform) 
     rescue
       puts "Error saving platform #{ platform }"
@@ -88,7 +105,7 @@ def annotations(name, cross_platform = false, &block)
 
         FileUtils.mkdir_p File.join("annotations", name)
         filename = File.join("annotations", name, dataset)
-        dataset += '_cross_platform' if cross_platform && MARQ::Platform::has_cross_platform?(platform)
+        dataset += MARQ::Name.cross_platform(dataset) if cross_platform && MARQ::Platform::has_cross_platform?(platform)
         next if ! MARQ::Dataset.exists?(dataset)
         terms = block.call(dataset)
         Open.write(filename, terms.to_yaml)

data/lib/MARQ/CustomDS.rb

@@ -42,25 +42,23 @@ module CustomDS
     Dir.glob(File.join(DATA_DIR, org) + '/*.orders').collect{|f| clean(File.basename(f.sub(/.orders/,'')))}.uniq
   end
 
-  def self.process_matrix(prefix, org)
-    conditions = Dir.glob(prefix + '/*').collect{|f| File.basename(f)} - %w(values codes info description cross_platform)
-    description = Open.read(File.join(prefix, 'description'))
-
-    info = YAML.load(File.open(File.join(prefix, 'info')))
-    r.CustomDS_process(prefix, false, conditions, description, info["two_channel"], !info["log2"])
-
-
-    codes = Open.read(File.join(prefix,'codes')).collect{|l| l.chomp}
-    cross_platform = ID.translate(org, codes) 
-
-    if cross_platform.length > codes.length / 4
-      Open.write(File.join(prefix,'cross_platform'),cross_platform.collect{|c| c || "NO MATCH"}.join("\n"))
+  def self.process_dataset(dataset, platform)
+    org    = platform_organism(platform)
+    platform_path = platform_path(platform)
+    prefix = File.join(DATA_DIR, org, MARQ::Name.clean(dataset))
+    
+    conditions  = Dir.glob(File.join(platform_path, '*')).collect{|f| File.basename(f)} - %w(values codes info description cross_platform)
+    description = Open.read(File.join(platform_path, 'description'))
+    info        = YAML.load(File.open(File.join(platform_path, 'info')))
+
+    if MARQ::Dataset.is_cross_platform?(dataset)
       r.CustomDS_process(prefix, true, conditions, description, info["two_channel"], !info["log2"])
+    else
+      r.CustomDS_process(prefix, false, conditions, description, info["two_channel"], !info["log2"])
     end
-  end
 
-  def self.process(name)
-   end
+ 
+  end
 
   def self.organisms
     Dir.glob(File.join(DATA_DIR, '*')).
@@ -69,23 +67,22 @@ module CustomDS
   end
 
   def self.dataset_path(dataset)
+
     organisms.each do |organism|
-      case
-      when File.exists?(File.join(DATA_DIR, organism, dataset + '.orders'))
+      if File.exists?(File.join(DATA_DIR, organism, dataset + '.orders')) || File.exists?(File.join(DATA_DIR, organism, dataset + '.skip'))
         return File.join(DATA_DIR, organism, dataset)
-      when File.exists?(File.join(DATA_DIR, organism, dataset + '.skip'))
-        return nil
       end
     end
+
     return nil
   end
 
   def self.platform_path(platform)
-    dataset_path(platform)
+    Dir.glob(File.join(DATA_DIR, '*', platform)).first
   end
 
   def self.platform_datasets(platform)
-    MARQ::Dataset.clean(platform)
+    MARQ::Name.clean(platform)
   end
 
   def self.platform_organism(platform)
@@ -104,22 +101,21 @@ module CustomDS
   end
 
   def self.organism_platforms(organism)
-    Dir.glob(File.join(DATA_DIR,organism,'*.orders')).
-      collect {|path| File.basename(path).sub(/\.orders$/,'').sub(/_cross_platform/,'')}.
-      uniq
+    Dir.glob(File.join(DATA_DIR, organism, '*', 'codes')).
+      collect {|path| File.basename(File.dirname(path))}.uniq
   end
 
   def self.process_platform(platform)
-  end
+    prefix = platform_path(platform)
+    org    = platform_organism(platform)
 
-  def self.process_dataset(dataset, platform = nil)
-    puts "Processing #{ dataset }"
-    org = dataset_organism(dataset)
-    prefix = File.join(DATA_DIR, org, dataset)
+    codes = Open.read(File.join(prefix,'codes')).collect{|l| l.chomp}
+    cross_platform = ID.translate(org, codes) 
 
-    CustomDS::process_matrix(prefix, org)
+    if cross_platform.length > codes.length / 4
+      Open.write(File.join(prefix,'cross_platform'),cross_platform.collect{|c| c || "NO MATCH"}.join("\n"))
+    end
   end
-
 end
 
 

data/lib/MARQ/GEO.rb

@@ -40,13 +40,11 @@ module GEO
 
     def self.platform_organism(platform)
       Open.read("http://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=#{platform}", :nice => @@nice).
-      match(%r#<td><a href="http://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi\?mode=Info&amp;id=\d+" onmouseout="onLinkOut\('HelpMessage' , geo_empty_help\)" onmouseover="onLinkOver\('HelpMessage' , geoaxema_organismus\)">(.*)</a></td>#)[1]
+        scan(%r#<a href="http://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi\?mode=Info&amp;id=\d+" onmouseout="onLinkOut\('HelpMessage' , geo_empty_help\)" onmouseover="onLinkOver\('HelpMessage' , geoaxema_organismus\)">(.*?)</a>#).collect{|p| p.first}.join(', ')
     end
 
   end
 
-
-
   # Parse information in .soft files
   module SOFT
 
@@ -111,22 +109,26 @@ module GEO
     end
 
 
+    #{{{ Parse soft files for several GEO entities
 
     def self.GSE(series)
       soft = get_soft(series)
 
+      # Find platform
       if match = soft.scan(/!Series_platform_id\s*=?\s*(.*)/)
         platform = match.flatten.collect{|p| p.strip}.join("_")
       else
         raise "No Platform information" 
       end
 
+      # Find title
       if soft.match(/!Series_title \s*=?\s*(.*)/)
         title = $1
       else
         raise "No Title information" 
       end
 
+      # Find summary
       if soft.match(/!Series_summary \s*=?\s*(.*)/)
         matches = soft.scan(/!Series_summary \s*=?\s*(.*)/).to_a
         description = matches.collect{|m| m.to_s.strip.sub(/!Series_summary \s*=?\s*/,'')}.join("\n")
@@ -134,6 +136,7 @@ module GEO
         raise "No Summary information" 
       end
 
+      # Find samples
       if soft.match(/!Series_sample_id \s*=?\s*(.*)/)
         matches = soft.scan(/!Series_sample_id \s*=?\s*(.*)/).to_a
         samples = matches.collect{|m| m.to_s.strip.sub(/!Series_sample_id \s*=?\s*/,'')}
@@ -152,6 +155,7 @@ module GEO
     def self.GSM(array)
       soft = get_soft(array)
 
+      # Find title
       if soft.match(/!Sample_title\s*=?\s*(.*)/)
         title = $1
       else
@@ -159,6 +163,7 @@ module GEO
       end
 
 
+      # Find description
       if soft.match(/!Sample_description \s*=?\s*(.*)/)
         description = $1
       else
@@ -173,6 +178,7 @@ module GEO
     end
 
     def self.GPL(platform)
+
       if !File.exist?(File.join(DATA_DIR, 'platforms',"#{platform}.yaml"))  &&
          !File.exist?(File.join(DATA_DIR, 'platforms',"#{platform}.skip")) 
         begin
@@ -263,6 +269,9 @@ module GEO
   # Use R to load and process the datasets 
   module Process
 
+    class PlatformNotProcessedError < StandardError; end
+    class AdhocPlatformCollisionError < StandardError; end
+
     # R library wrapper
     module R
       @@r = nil
@@ -350,109 +359,87 @@ module GEO
         rearange(platform_positions, prefix + '.' + ext)
       }
 
+      FileUtils.cp(platform_codes_file, prefix + '.codes')
       Open.write(prefix + '.swap', platform_positions.join("\n"))
-    end
-
-
-    # Process a dataset. Need to specify the platform. The field parameter can
-    # be used to use a different column for the field. 
-    #
-    # Deprecated in favor of using the original firt column and using a
-    # different one only for translation
-    def self.GDS(dataset, platform, field = nil)
-      puts "Processing GDS #{ dataset }. Platform #{ platform }"
-      platform_path = GEO.platform_path(platform)
 
-      puts "-- Original"
-      prefix = File.join(platform_path, 'GDS', dataset.to_s)
-      R.GDS(dataset, prefix, field, nil)
-
-      # Was there an error?
-      if File.exist?(prefix + '.skip')
-        FileUtils.cp(prefix + '.skip', prefix + '_cross_platform.skip')
-        return
-      end
-
-      if File.exist?(File.join(platform,'cross_platform'))
-        puts "-- Translated to cross_platform format"
-        R.GDS(dataset, prefix + '_cross_platform', field, File.join(platform_path, 'translations')) 
-      else
-        puts "No cross_platform probe ids for platform"
-      end
     end
 
-    # Process a series. The info parameters is a hash with the :array,
-    # :platform, :log2 and :fields keys
     def self.GSE(series, info)
-      return if Dir.glob(File.join(info[:platform], 'GSE', series) + '.*').any?
-
-
+      platform = info[:platform]
+      do_log   = info[:log2].nil? ? nil : !info[:log2]
+      fields   = info[:fields]
+ 
+      # Determine samples and sample conditions
       gsms = []
       conditions = {}
       info[:arrays].each{|gsm, cond|
         gsms << gsm
-        cond.each{|condition, value|
-          conditions[condition] ||= []
-          conditions[condition] << value
+        cond.each{|type, value|
+          conditions[type] ||= []
+          conditions[type] << value
         }
       }
-      platform = info[:platform]
-      do_log = nil
-      do_log = !info[:log2] if info[:log2]
-      fields = info[:fields]
+     
+      # Adhoc platforms are for series with samples from different platforms.
+      # They are created when the series is processed
+      adhoc_platform = platform.match(/_/) != nil
 
-      platform_path = GEO::platform_path(platform)
-      return if platform_path.nil?
-      prefix = File.join(platform_path, 'GSE', series.to_s)
+      raise PlatformNotProcessedError if ! adhoc_platform && ! MARQ::Platform.exists?(platform)
+      raise AdhocPlatformCollisionError if adhoc_platform && MARQ::Platform.exists?(platform)
 
-      puts "Processing GSE #{ series }. Platform #{ platform }"
-      puts "-- Original"
-      R.GSE(gsms, conditions, do_log, prefix, nil, fields, info[:title], info[:description])
+      cross_platform = MARQ::Name.is_cross_platform?(series)
 
-      # Was there an error?
-      if File.exist?(prefix + '.skip')
-        FileUtils.cp(prefix + '.skip', prefix + '_cross_platform.skip')
-        return
-      end
+      platform_path = GEO.platform_path(platform)
 
-      if platform =~ /_/
-        FileUtils.cp(prefix + '.codes', File.join(platform_path,'codes'))
-        codes  = Open.read(File.join(platform_path, 'codes')).collect{|l| l.chomp}
-        organism = SOFT::GPL(platform.match(/(.*?)_/)[1])[:organism]
-        translations = translate(organism, codes) 
-        Open.write(File.join(platform_path, 'translations'), translations.collect{|v| v || "NO MATCH"}.join("\n"))
-        Open.write(File.join(platform_path, 'cross_platform'), translations.compact.sort.uniq.join("\n"))
-      else
-        # Are the codes of the series equivalent to the ones in the platform?
-        if  File.open(File.join(platform_path,'codes')).collect{|l| l.chomp} != File.open(prefix + '.codes').collect{|l| l.chomp}
+      prefix = File.join(platform_path, 'GSE', series)
+
+      FileUtils.rm(prefix + '.skip') if File.exist?(prefix + '.skip')
+
+      if ! cross_platform
+        R.GSE(gsms, conditions, do_log, prefix, nil, fields, info[:title], info[:description])
+
+        # Set up codes and cross_platform for adhoc platforms
+        if adhoc_platform 
+          codes        = Open.read(prefix + '.codes').collect{|l| l.chomp}
+          organism     = GEO.platform_organism(platform.split(/_/)[0])
+          translations = translate(organism, codes) 
+          FileUtils.cp(prefix + '.codes', File.join(platform_path,'codes'))
+          Open.write(File.join(platform_path, 'translations'), translations.collect{|v| v || "NO MATCH"}.join("\n"))
+          Open.write(File.join(platform_path, 'cross_platform'), translations.compact.sort.uniq.join("\n"))
+        else
           fix_GSE_ids(File.join(platform_path, 'codes'),prefix);
-          FileUtils.cp(File.join(platform_path, 'codes'),prefix + '.codes')
         end
+
+      else
+        R.GSE(gsms, conditions, do_log, prefix, File.join(platform_path, 'translations'), fields, info[:title], info[:description])
+        fix_GSE_ids(File.join(platform_path, 'cross_platform'),prefix);
       end
 
+    end
 
-      if File.exist?(File.join(platform,'translations'))
-        FileUtils.cp(File.join(platform,'translations'), prefix + '.translations')
-        if File.exist?(prefix + '.swap')
-          orders = Open.read(prefix + '.swap').collect{|l| l.chomp}
-          inverse_orders = Array.new(orders.length)
-          orders.each_with_index{|pos,i|
-            next if pos !~ /\d/
-              inverse_orders[pos.to_i] = i
-          }
-          rearange(inverse_orders, prefix + '.translations', "NO MATCH")
-        end
-        puts "-- Translated to cross_platform format"
-        R.GSE(gsms, conditions, do_log, prefix + '_cross_platform', prefix + '.translations',fields, info[:title], info[:description])
-        fix_GSE_ids(File.join(platform_path, 'cross_platform'),prefix + '_cross_platform');
-        FileUtils.cp(File.join(platform_path, 'cross_platform'),prefix + '_cross_platform.codes')
-        FileUtils.rm(prefix + '.translations') if File.exist?(prefix + '.translations')
+    # Process a dataset. Need to specify the platform. The field parameter can
+    # be used to use a different column for the field. 
+    #
+    # Deprecated in favor of using the original firt column and using a
+    # different one only for translation
+    def self.GDS(dataset, platform, field = nil)
+      raise PlatformNotProcessedError if ! MARQ::Platform.exists? platform
+
+      cross_platform = MARQ::Name.is_cross_platform? dataset
+
+      platform_path = GEO.platform_path(platform)
+      prefix = File.join(platform_path, 'GDS', dataset)
+
+      FileUtils.rm(prefix + '.skip') if File.exist?(prefix + '.skip')
+
+      if cross_platform
+        R.GDS(MARQ::Name.clean(dataset), prefix, field, File.join(platform_path, 'translations'))
       else
-        puts "No cross_platform probe ids for platform"
+        R.GDS(dataset, prefix, field, nil) 
       end
-      FileUtils.rm(prefix + '.swap') if File.exist?(prefix + '.swap')
     end
 
+
     # Load GPL data. Translates IDS of the platform probes using AILUN and our
     # system (called biomart for clarity)
     def self.GPL(platform)
@@ -535,7 +522,6 @@ module GEO
       end
 
     end
-
   end
 
   def self.platforms
@@ -554,7 +540,7 @@ module GEO
 
   def self.platform_path(platform)
     path = File.join(DATA_DIR, platform)
-    path = nil unless File.exists? path
+    path = nil unless File.exists? File.join(path, 'codes')
     path
   end
 
@@ -567,15 +553,15 @@ module GEO
 
     platforms.each do |platform|
       platform_path = platform_path(platform)
+
       next if platform_path.nil?
 
       prefix = File.join(platform_path, dataset_type(dataset).to_s, dataset)
-      case
-      when File.exists?(prefix + '.orders')
+      
+      if File.exists?(prefix + '.orders') || File.exists?(prefix + '.skip')
         return File.join(platform_path, dataset_type(dataset).to_s, dataset)
-      when File.exists?(prefix + '.skip')
-        return nil
       end
+
     end
 
     return nil
@@ -584,7 +570,7 @@ module GEO
   def self.platform_datasets(platform)
     cross_platform = MARQ::Platform.is_cross_platform? platform
     
-    path = platform_path(MARQ::Platform.clean(platform))
+    path = platform_path(MARQ::Name.clean(platform))
     return [] if path.nil?
 
     datasets = Dir.glob(File.join(path, '*', '*.orders')).
@@ -592,7 +578,7 @@ module GEO
 
     if cross_platform
       datasets.select  {|dataset| MARQ::Dataset.is_cross_platform? dataset }.
-               collect {|dataset| MARQ::Dataset.clean(dataset) }
+               collect {|dataset| MARQ::Name.clean(dataset) }
     else
       datasets.select  {|dataset| ! MARQ::Dataset.is_cross_platform? dataset }
     end
@@ -614,7 +600,7 @@ module GEO
   end
 
   def self.process_platform(platform)
-    GEO::Process.GPL(platform)
+    GEO::Process.GPL(platform) unless platform =~ /_/
   end
 
   def self.process_dataset(dataset, platform)