rbbt-marq 1.0.9 → 1.1.0

data/R/GEO.R

@@ -1,106 +1,8 @@
 library(Biobase);
 library(GEOquery);
 
-
-GEO.path <- function(dataset, cross_platform = FALSE, datadir = NULL){
-    if (is.null(datadir) && exists('MARQ.config')){
-       datadir= paste(MARQ.config$datadir, 'GEO', sep="/");
-    }
-
-    if (is.null(datadir)){
-        print("No datadir specified and no default found (MARQ.config$datadir");
-        exit(-1);
-    }
-
-    if ( length(grep('_cross_platform', dataset)) == 0 && cross_platform){
-        dataset = paste(dataset, '_cross_platform', sep = "");
-    
-    }
-    
-
-    files = Sys.glob(paste(datadir,'*', '*', paste(dataset, 'orders', sep="."), sep="/"));
-    
-    if (length(files) == 0){
-        return(NULL);
-    }
-    else{
-        return(sub('.orders','', files[1]));
-    }
-}
-
-GEO.platform <- function(dataset, datadir = NULL){
-
-    path = GEO.path(dataset, datadir = datadir);
-
-    if (is.null(path)){ return(NULL);}
-
-    return(sub(".*(GPL\\d+).*","\\1", path, perl = TRUE));
-}
-
-GEO.platform.path <- function(platform, datadir = NULL){
-    if (is.null(datadir) && exists('MARQ.config')){
-       datadir= paste(MARQ.config$datadir, 'GEO', sep="/");
-    }
-
-    if (is.null(datadir)){
-        print("No datadir specified and no default found (MARQ.config$datadir");
-        exit(-1);
-    }
-
-    return(paste(datadir, platform, sep="/"));
-}
-
-GEO.platform.datasets <- function(platform, cross_platform = TRUE, series = TRUE, datadir = NULL){
-    if (cross_platform){
-        cp.suffix = '_cross_platform'
-    }
-    else{
-        cp.suffix = ''
-    }
-
-    if (series){
-        pattern = '*'
-    }
-    else{
-        pattern = 'GDS'
-    }
-    files = Sys.glob(paste(GEO.platform.path(platform, datadir), pattern, paste('*',cp.suffix,'.orders',sep=""),sep="/"))
-
-    return(sapply(files, function(path){ sub(".*((?:GDS|GSE)\\d+).*", '\\1', path, perl=TRUE)}, USE.NAMES = FALSE));
-}
-
-GEO.values <- function(data){
-    values <- MA.process(data$m, data$conditions, data$two.channel)
-
-    if (length(values$ratios) == 0){
-        return(NULL);
-    }else{
-        ratios   = as.data.frame(values$ratios);
-        t        = as.data.frame(values$t);
-        p.values = as.data.frame(values$p.values);
-
-
-        # Calculate orders from best information
-        best = vector();
-        names = vector();
-        for (name in colnames(ratios)){
-            if (sum(colnames(t) == name) > 0){
-                best = cbind(best, t[,name]);
-                names = c(names, name);
-            }else{
-                best = cbind(best, ratios[,name]);
-                names = c(names, paste(name,'[ratio]', sep=" "));
-            }
-        }
-        rownames(best) <- rownames(ratios)
-        orders   = as.data.frame(MA.get_order(best));
-        colnames(orders) <- names
-
-        return(list(ratios = ratios, t = t, p.values = p.values, orders = orders));
-    }
-}
-
-
+####################################################
+# Data retrieval functions
 
 GEO.get <- function(name, cachedir = NULL){
     if (is.null(cachedir) && exists('MARQ.config')){
@@ -161,7 +63,10 @@ GEO.GDS.data <- function(name, id.field = NULL, translation.file = NULL, cachedi
     if (!is.null(id.field)){
         trans = featureData(eSet)[[id.field]];
     }
-    if (!is.null(translation.file)){
+    if (!is.null(translation.file) && translation.file != FALSE){
+        if (translation.file == TRUE && exists(MARQ.platform.path)){
+           translation.file = paste(MARQ.platform.path(gpl_name), 'translations', sep="/");
+        }
         trans = scan(file=translation.file,what=character(),sep="\n",quiet=T);
     }
 
@@ -169,33 +74,12 @@ GEO.GDS.data <- function(name, id.field = NULL, translation.file = NULL, cachedi
         m <- MA.translate(m, trans);
     }
 
-    return (list(conditions = conditions, m = m, two.channel = two.channel, description = description))
-}
 
 
-GEO.GDS.process <- function(name, prefix, id.field = NULL, translation.file = NULL,cachedir=NULL){
-    tryCatch(
-    {
-        gds.data = GEO.GDS.data(name, id.field, translation.file, cachedir)
-        values = GEO.values(gds.data)
-        if (is.null(values)){
-            write(file=paste(prefix,'skip',sep="."), "No suitable samples for analysis" );
-        }else{
-            MA.save(prefix, values$orders, values$ratios, values$t, values$p.values, colnames(values$orders), gds.data$description);
-        }
-    }
-    ,
-        error=function(x){ 
-             print("Exception caught");
-             print(x);
-             write(file=paste(prefix,'skip',sep="."), paste("An exception was caught during the analysis.",x,sep="\n") );
-        }
-    )
+    return (list(conditions = conditions, m = m, two.channel = two.channel, description = description))
 }
 
 
-
-
 GEO.GSE.data <- function(gsms, conditions, do.log2 = NULL, translation.file = NULL, use.fields = NULL, cachedir = NULL){
 
     c = sapply(conditions,function(x){x});
@@ -261,7 +145,10 @@ GEO.GSE.data <- function(gsms, conditions, do.log2 = NULL, translation.file = NU
     }
 
     trans = NULL
-    if (!is.null(translation.file)){
+    if (!is.null(translation.file) && translation.file != FALSE){
+        if (translation.file == TRUE && exists(MARQ.platform.path)){
+           translation.file = paste(MARQ.platform.path(gpl_name), 'translations', sep="/");
+        }
         trans = read.table(file=translation.file, sep="\t",header=F)[,1];
     }
     if (!is.null(trans)){
@@ -282,6 +169,63 @@ GEO.GSE.data <- function(gsms, conditions, do.log2 = NULL, translation.file = NU
     return (list(conditions = conditions, m = m, two.channel = two.channel))
 }
 
+
+####################################################
+# Processing Functions
+
+GEO.values <- function(data){
+    values <- MA.process(data$m, data$conditions, data$two.channel)
+
+    if (length(values$ratios) == 0){
+        return(NULL);
+    }else{
+        ratios   = as.data.frame(values$ratios);
+        t        = as.data.frame(values$t);
+        p.values = as.data.frame(values$p.values);
+
+
+        # Calculate orders from best information
+        best = vector();
+        names = vector();
+        for (name in colnames(ratios)){
+            if (sum(colnames(t) == name) > 0){
+                best = cbind(best, t[,name]);
+                names = c(names, name);
+            }else{
+                best = cbind(best, ratios[,name]);
+                names = c(names, paste(name,'[ratio]', sep=" "));
+            }
+        }
+        rownames(best) <- rownames(ratios)
+        orders   = as.data.frame(MA.get_order(best));
+        colnames(orders) <- names
+
+        return(list(ratios = ratios, t = t, p.values = p.values, orders = orders));
+    }
+}
+
+
+
+GEO.GDS.process <- function(name, prefix, id.field = NULL, translation.file = NULL,cachedir=NULL){
+    tryCatch(
+    {
+        gds.data = GEO.GDS.data(name, id.field, translation.file, cachedir)
+        values = GEO.values(gds.data)
+        if (is.null(values)){
+            write(file=paste(prefix,'skip',sep="."), "No suitable samples for analysis" );
+        }else{
+            MA.save(prefix, values$orders, values$ratios, values$t, values$p.values, colnames(values$orders), gds.data$description);
+        }
+    }
+    ,
+        error=function(x){ 
+             print("Exception caught");
+             print(x);
+             write(file=paste(prefix,'skip',sep="."), paste("An exception was caught during the analysis.",x,sep="\n") );
+        }
+    )
+}
+
 GEO.GSE.process <- function(gsms, conditions, prefix, do.log2 = NULL, translation.file = NULL, use.field = NULL, title = NULL,  description = NULL,cachedir=NULL){
     tryCatch(
     {

data/R/GEOquery_patch.R

@@ -0,0 +1,44 @@
+library(GEOquery);
+
+# The original version of the function failed if the dataset had extra probe ids not in the platform.
+# This version fixes that
+"GDS2eSet" <-
+  function(GDS,do.log2=FALSE,GPL=NULL,AnnotGPL=TRUE) {
+    require(Biobase)
+                                        # exclude non-numeric columns
+    if(is.null(GPL)) {
+      GPL <- getGEO(Meta(GDS)$platform,AnnotGPL=AnnotGPL)
+    }
+    ord.table <- match(Table(GDS)[,1],Table(GPL)[,1])
+    inc.columns <- grep('GSM',colnames(Table(GDS)))
+    mat <- suppressWarnings(as.matrix(apply(Table(GDS)[,inc.columns],2,
+                                            function(x) {as.numeric(as.character(x))})))
+    if(do.log2) {
+      expr <- log2(mat)
+    } else {
+      expr <- mat
+    }
+    rownames(expr) <- as.character(Table(GDS)$ID_REF)
+    tmp <- Columns(GDS)
+    rownames(tmp) <- as.character(tmp$sample)
+    pheno <- new("AnnotatedDataFrame",data=tmp)
+    mabstract=ifelse(is.null(Meta(GDS)$description),"",Meta(GDS)$description)
+    mpubmedids=ifelse(is.null(Meta(GDS)$pubmed_id),"",Meta(GDS)$pubmed_id)
+    mtitle=ifelse(is.null(Meta(GDS)$title),"",Meta(GDS)$title)
+    dt <- Table(GPL)
+    rownames(dt) <- as.character(dt$ID)
+    featuredata <- new('AnnotatedDataFrame',data=dt[ord.table,],
+                       varMetadata=data.frame(Column=Columns(GPL)[,1],
+                         labelDescription=Columns(GPL)[,2]))
+
+    # use !is.na(ord.table) to remove extra probe ids in GDS and not in GPL
+    eset <- new('ExpressionSet',exprs=expr[!is.na(ord.table),],phenoData=pheno,
+                featureData=featuredata[!is.na(ord.table),],
+                experimentData=new("MIAME",
+                  abstract=mabstract,
+                  title=mtitle,
+                  pubMedIds=mpubmedids,
+                  other=Meta(GDS)))
+    return(eset)
+  }
+

data/R/MARQ.R

@@ -1,3 +1,79 @@
 library('yaml');
 
 MARQ.config = yaml.load_file('~/.MARQ');
+
+
+####################################################
+# GEO platforms and datasets helper functions
+
+MARQ.GEO.path <- function(dataset, cross_platform = FALSE, datadir = NULL){
+    if (is.null(datadir) && exists('MARQ.config')){
+       datadir= paste(MARQ.config$datadir, 'GEO', sep="/");
+    }
+
+    if (is.null(datadir)){
+        print("No datadir specified and no default found (MARQ.config$datadir");
+        exit(-1);
+    }
+
+    if ( length(grep('_cross_platform', dataset)) == 0 && cross_platform){
+        dataset = paste(dataset, '_cross_platform', sep = "");
+    
+    }
+    
+
+    files = Sys.glob(paste(datadir,'*', '*', paste(dataset, 'orders', sep="."), sep="/"));
+    
+    if (length(files) == 0){
+        return(NULL);
+    }
+    else{
+        return(sub('.orders','', files[1]));
+    }
+}
+
+MARQ.GEO.platform <- function(dataset, datadir = NULL){
+
+    path = MARQ.GEO.path(dataset, datadir = datadir);
+
+    if (is.null(path)){ return(NULL);}
+
+    return(sub(".*(GPL\\d+).*","\\1", path, perl = TRUE));
+}
+
+MARQ.GEO.platform.path <- function(platform, datadir = NULL){
+    if (is.null(datadir) && exists('MARQ.config')){
+       datadir= paste(MARQ.config$datadir, 'GEO', sep="/");
+    }
+
+    if (is.null(datadir)){
+        print("No datadir specified and no default found (MARQ.config$datadir");
+        exit(-1);
+    }
+
+    return(paste(datadir, platform, sep="/"));
+}
+
+MARQ.GEO.platform.datasets <- function(platform, cross_platform = TRUE, series = TRUE, datadir = NULL){
+    if (cross_platform){
+        cp.suffix = '_cross_platform'
+    }
+    else{
+        cp.suffix = ''
+    }
+
+    if (series){
+        pattern = '*'
+    }
+    else{
+        pattern = 'GDS'
+    }
+    files = Sys.glob(paste(MARQ.GEO.platform.path(platform, datadir), pattern, paste('*',cp.suffix,'.orders',sep=""),sep="/"))
+
+    return(sapply(files, function(path){ sub(".*((?:GDS|GSE)\\d+).*", '\\1', path, perl=TRUE)}, USE.NAMES = FALSE));
+}
+
+MARQ.GEO.load <- function(dataset, cross_platform = FALSE, orders = TRUE, logratios = TRUE, t = TRUE, p.values = TRUE){
+    return(MA.load(MARQ.GEO.path(dataset, cross_platform), orders, logratios, t, p.values));
+}
+

data/install_scripts/GEO/Rakefile

@@ -5,71 +5,144 @@ require 'MARQ/ID'
 require 'yaml'
 require 'progress-monitor'
 require 'MARQ/MADB'
+require 'rbbt/sources/organism'
 
 $platform       ||= ENV['platform']
-$org            ||= [$organism, ENV['organism'], nil].reject{|e| e.nil?}.first 
-$series         ||= ENV['series']
+$organism       ||= [$organism, ENV['organism'], nil].compact.first 
+$dataset        ||= ENV['dataset']
 
 $expr_threshold ||= (ENV['threshold'] || 0.05).to_f
 $folds          ||= (ENV['folds'] || 2.5).to_f
 $nth_genes      ||= (ENV['nth_genes'] || 100).to_i
 
-$force       = [$force, ENV['force'], false].         reject{|e| e.nil?}.first 
-$update_db   = [$update_db, ENV['update_db'], true].  reject{|e| e.nil?}.first 
-$fdr         = [$fdr, ENV['fdr'], true].              reject{|e| e.nil?}.first 
-$do_folds    = [$do_folds, ENV['do_folds'], true].    reject{|e| e.nil?}.first 
+$force       = [$force, ENV['force'], false].compact.first.to_s == 'true'  
+$tranlations = [$tranlations, ENV['translations'], false].compact.first.to_s == 'true'  
+$series      = [$series, ENV['series'], true].compact.first.to_s == 'true' 
+$update_db   = [$update_db, ENV['update_db'], false].compact.first.to_s == 'true'  
+$skip_db     = [$skip_db, ENV['skip_db'], false].compact.first.to_s == 'true'  
+$fdr         = [$fdr, ENV['fdr'], true].compact.first.to_s == 'true'  
+$do_folds    = [$do_folds, ENV['do_folds'], true].compact.first.to_s == 'true'  
 
 
 # Record changes in order to update DB
 $changes = false
-module GEO
+module GEO::Process::R
   class << self
-    alias_method :get_GDS_old, :get_GDS
-    def get_GDS(*args)
+    alias_method :GDS_old, :GDS
+    def GDS(*args)
       $changes = true
-      get_GDS_old(*args)
+      GDS_old(*args)
     end
   
-    alias_method :get_GSE_old, :get_GSE
-    def get_GSE(*args)
+    alias_method :GSE_old, :GSE
+    def GSE(*args)
       $changes = true
-      get_GSE_old(*args)
+      GSE_old(*args)
     end
   end
 end
 
 
-def process_platform(platform, update_db = false)
-  begin
-    $changes = false
-    GEO.process_platform(platform)
-    GEO.process_platform_datasets(platform, $force && !update_db)
-    if update_db && ($changes || $force)
-      puts "Saving #{platform}"       
-      MADB::GEO.saveGPL(platform) if update_db && ($changes || $force)
-    end
-  rescue
-    puts $!.message
+def process_list
+  return {GEO::Remote::dataset_platform($dataset) => [$dataset]} if $dataset
+  return {$platform => GEO::Remote::platform_datasets($platform)} if $platform
+  
+  if $organism
+    organisms = [$organism]
+  else
+    organisms = Organism.all(true)
+  end
+
+  list = {}
+  organisms.each{|organism|
+    GEO::Remote::organism_platforms(organism).each{|platform|
+      datasets = GEO::Remote::platform_datasets(platform)
+
+      # Platforms with no datasets are skiped, although if they have series
+      # they may be considered later
+      next if datasets.empty?
+
+      if $tranlations || $update_db
+        list[platform] = []
+      else
+        if ! $force
+          datasets.reject!{|dataset| Dir.glob(File.join(GEO.dataset_path(dataset, platform) || "MISSING", '.*')).any? }
+        end
+        list[platform] = datasets
+      end
+    }
+  }
+
+  if $series && ! $translations 
+    series = Dir.glob('series/*.yaml').collect{|f| File.basename(f).sub(/.yaml/,'')}               
+    series.each{|serie|
+      platform = GEO::SOFT.GSE(serie)[:platform]
+      begin
+        if organisms.include? GEO::SOFT::GPL(platform)[:organism].to_s
+          list[platform] ||= []
+          list[platform] << serie unless $tranlations || $update_db    
+        end
+      rescue
+        puts "Error process series #{serie} platform #{platform}"
+        puts $!.message
+        puts $!.backtrace.join("\n")
+      end
+    }
   end
+
+  return list
 end
 
-def process_serie(serie, update_db = false)
-  begin
+desc "Analyze datasets"
+task 'data' do 
+
+  platforms_to_save = []
+
+  process_list.each{|platform, datasets|
+
+    begin
+      # Prepare the platform
+      GEO::Process.GPL(platform)
+    rescue
+      puts "Error processing platform #{platform}"
+      puts $!.message
+      puts $!.backtrace.join("\n")
+      next
+    end
+
     $changes = false
-    info = YAML::load(File.open("series/#{ serie }.yaml"))
+    # Process all datasets
+    datasets.each{|dataset|
+      begin
+        if dataset =~ /GDS/
+          GEO::Process.GDS(dataset, platform)
+        else
+          info = YAML::load(File.open("series/#{ dataset }.yaml"))
+          FileUtils.rm("platforms/#{ info[:platform] }.skip") if File.exist?  "platforms/#{ info[:platform] }.skip"
+          GEO::Process.GSE(dataset, info)
+        end
+
+        # Mark the platform for saving in DB
+      rescue
+        puts "Error processing dataset #{ dataset }"
+        puts $!.message
+        puts $!.backtrace.join("\n")
+      end
+    }
 
-    FileUtils.rm("platforms/#{ info[:platform] }.skip") if File.exist?  "platforms/#{ info[:platform] }.skip"
-    GEO.process_platform(info[:platform]) if !File.exist? info[:platform]
+    platforms_to_save << platform if $changes || $update_db
+  }
 
-    GEO.process_GSE(serie, info)
-    if info[:platform] && update_db
-      puts "Saving #{info[:platform]}"
-      MADB::GEO.saveGPL(info[:platform]) if info[:platform] && update_db
+  platforms_to_save.each{|platform|
+    begin
+      puts "Saving #{platform}"
+      MADB::GEO.saveGPL(platform) 
+    rescue
+      puts "Error saving platform #{ platform }"
+      puts $!.message
+      puts $!.backtrace.join("\n")
     end
-    info[:platform]
-  rescue
-    puts $!.message
-  end
+  }
 end
 
 def annotations(name, cross_platform = false, &block)
@@ -98,7 +171,7 @@ def annotations(name, cross_platform = false, &block)
     end
   }
   return if $platform
-  
+
   Progress.monitor("Annotating with #{ name }, series")
   series = Dir.glob('series/*.yaml').collect{|f| File.basename(f).sub(/.yaml/,'')}
   series.each{|serie|
@@ -125,42 +198,6 @@ def goterms(org, list, slim, threshold)
 end
 
 
-task 'data' do
-  if $platform
-    process_platform($platform, $update_db)
-  elsif $series
-    process_serie($series, $update_db)
-  elsif $org
-    org = $org
-    platforms = GEO::Eutils.organism_platforms(org).select{|platform|  GEO::Eutils.GPL_datasets(platform).any?}
-    Progress.monitor("Platforms for #{Organism.name(org)}")
-    platforms.each{|platform|
-      puts "Platform #{ platform }"
-      process_platform(platform,$update_db)
-    }
-  else
-    # Series
-    series = Dir.glob('series/*.yaml').collect{|f| File.basename(f).sub(/.yaml/,'')}
-    platform2save = []
-    series.each{|serie|
-      platform  = process_serie(serie, false)
-      platform2save << platform if $update_db && ($changes || $force)
-    }
-
-    Progress.monitor("Saving #{platform2save.uniq.length} platforms: ")
-    platform2save.uniq.each{|platform| MADB::GEO.saveGPL(platform) }
-
-    # Platforms
-    for org in Organism.all
-      platforms = GEO::Eutils.organism_platforms(org).select{|platform|  GEO::Eutils.GPL_datasets(platform).any?}
-      Progress.monitor("Platforms for #{Organism.name(org)}")
-      platforms.each{|platform|
-        puts "Platform #{ platform }"
-        process_platform(platform,$update_db)
-      }
-    end
-  end
-end
 
 task 'annotate_Words' do
   require 'MARQ/annotations'