rbbt-marq 1.1.0 → 2.0.0

data/R/GEO.R

@@ -9,17 +9,19 @@ GEO.get <- function(name, cachedir = NULL){
        cachedir = paste(MARQ.config$cachedir, 'GEO', sep="/");
     }
 
-    if (is.null(cachedir)){
+    if (is.null(cachedir) || cachedir == FALSE){
         object  <- getGEO(name);
     }else{
-        filename = dir(cachedir,pattern= paste(name, 'soft', sep='.'))[1]
-        if (is.na(filename)){
+        filename = dir(cachedir,pattern= paste(name, 'soft', sep='.'))[1];
+        complete_path = paste(cachedir,filename,sep="/");
+        if (is.na(filename) || file.info(complete_path)$size == 0){
             object  <- getGEO(name, destdir=cachedir);
         }else{
-            object  <- getGEO(name, filename = paste(cachedir,filename,sep="/"));
+            object  <- getGEO(name, filename = complete_path);
         }
     }
 
+
     object
 }
 

data/R/MA.R

@@ -307,6 +307,7 @@ MA.process <- function(m, conditions_list, two.channel = FALSE){
 
 MA.save <- function(prefix, orders, ratios, t , p.values, experiments, description = NULL) {
     if (is.null(orders)){
+        cat("No suitable samples for analysis\n")
         write(file=paste(prefix,'skip',sep="."), "No suitable samples for analysis" );
     } else {
         write.table(file=paste(prefix,'orders',sep="."), orders, sep="\t",  row.names=F, col.names=F, quote=F);

data/bin/marq_config

@@ -88,7 +88,7 @@ $USAGE =<<EOT
 EOT
 
 class Controller < SimpleConsole::Controller
-  params :string => {:p => :platform, :s => :series,  :o => :organism}, :boolean => {:d => :update_db, :f => :force}, :integer => {:p => :port}
+  params :string => {:t => :target, :p => :platform, :s => :series,  :o => :organism}, :boolean => {:d => :update_db, :f => :force}, :integer => {:p => :port}
 
 
   def prepare
@@ -103,6 +103,7 @@ class Controller < SimpleConsole::Controller
     $update_db = params[:update_db] && params[:update_db] =~ /true|yes|y/i
     $force     = params[:force]     && params[:force]     =~ /true|yes|y/i
     @actions   = params[:id] || %w(GEO)
+    @rake_action  = params[:target] || 'default' 
   end
 
   def config
@@ -198,10 +199,10 @@ class View < SimpleConsole::View
 
     @actions.each{|action|
       Thread.new{
-        puts "Updating #{action}"
+        puts "Updating #{action}. Target: #{@rake_action}"
         FileUtils.cd File.join(MARQ.datadir, action)
         load File.join(MARQ.datadir, action, 'Rakefile')
-        Rake::Task['default'].invoke
+        Rake::Task[@rake_action].invoke
       }.join
     }
   end

data/install_scripts/CustomDS/Rakefile

@@ -1,223 +1,35 @@
 require 'MARQ'
+require 'MARQ/main'
 require 'MARQ/CustomDS'
 require 'MARQ/MADB'
 require 'MARQ/annotations'
 require 'progress-monitor'
-
-$org            ||= ENV['organism']
-
-$expr_threshold ||= (ENV['threshold'] || 0.05).to_f
-$folds          ||= (ENV['folds'] || 2.5).to_f
-$nth_genes      ||= (ENV['nth_genes'] || 100).to_i
-
-$force       = [$force, ENV['force'], false].         reject{|e| e.nil?}.first 
-$update_db   = [$update_db, ENV['update_db'], true].  reject{|e| e.nil?}.first 
-$fdr         = [$fdr, ENV['fdr'], true].              reject{|e| e.nil?}.first 
-$do_folds    = [$do_folds, ENV['do_folds'], true].    reject{|e| e.nil?}.first 
-
-
-
-
-task 'data' do
-  organisms = Dir.glob(CustomDS::customdir + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - ["annotations"]
-  
-  organisms.each{|org|
-    datasets = Dir.glob(File.join(CustomDS::customdir, org) + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)}
-    if $force
-      missing = datasets
-    else
-      missing = datasets - CustomDS::datasets(org)
-    end
-    missing.each{|d|
-      CustomDS::process(d)
-      MADB::CustomDS::save(d)
-    }
-  }
-end
-
-
-def annotations(name, cross_platform = false, &block)
-  FileUtils.mkdir_p File.join("annotations", name) unless File.exist? File.join("annotations", name)
-  organisms = Dir.glob(CustomDS::customdir + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - ["annotations"]
-
-  Progress.monitor("Annotating with #{ name }")
-  organisms.each{|org|
-    Progress.monitor("Annotating organism #{ org }")
-    CustomDS::datasets(org).each{|dataset|
-      begin
-        next if File.exist? File.join("annotations", name, dataset)
-        filename = File.join("annotations", name, dataset)
-        dataset += '_cross_platform' if cross_platform && GEO.has_cross_platform?(dataset)
-        next if File.exist? CustomDS.path(dataset) + '.skip'
-        terms = block.call(dataset)
-        Open.write(filename, terms.to_yaml)
-      rescue Exception
-        puts $!.message
-      end
-    }
-  }
-end
-
-def goterms(org, list, slim, threshold)
-  return [] if list.empty?
-  results = Annotations::GO::Genecodis::Local.analysis(org, list, slim)
-  results.
-    select{|info| info[:s].to_i  > 2 }.
-    select{|info| info[:hyp_c].to_f < threshold }.
-    collect{|info| info[:items]}.collect{|id| GO::id2name(id)}
-end
-
-
-task 'annotate_Words' do
-  FileUtils.mkdir_p "annotations/Words"
-  require 'rbbt/bow/bow'
-  organisms = Dir.glob(CustomDS::customdir + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - ["annotations"]
-  organisms.each{|org|
-    CustomDS::datasets(org).each{|dataset|
-      begin
-        next if File.exist? "annotations/Words/#{ dataset }"
-        next if File.exist? CustomDS.path(dataset) + '.skip'
-
-        terms = {}
-        description = Open.read(CustomDS.path(dataset) + '.description')
-        terms[:dataset] = [dataset] +  description.words
-        Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
-          name = name.strip
-          terms[name] = name.sub(/\[ratio\]/,'').words
-        }
-        
-        Open.write("annotations/Words/#{ dataset }", terms.to_yaml)
-
-      rescue Exception
-        puts $!.message
-      end
-    }
- }
-end
-
-task 'annotate_UMLS' do
-  FileUtils.mkdir_p "annotations/UMLS"
-  organisms = Dir.glob(CustomDS::customdir + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - ["annotations"]
-  organisms.each{|org|
-    CustomDS::datasets(org).each{|dataset|
-      begin
-        next if File.exist? "annotations/UMLS/#{ dataset }"
-        next if File.exist? CustomDS.path(dataset) + '.skip'
-        terms = {}
-        description = Open.read(CustomDS.path(dataset) + '.description')
-        terms[:dataset] = Annotations::UMLS::OBA(description)
-        Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
-          name = name.strip
-          terms[name] = Annotations::UMLS::OBA(name)
-        }
-        Open.write("annotations/UMLS/#{ dataset }", terms.to_yaml)
-      rescue Exception
-        puts $!.message
+require File.join(File.dirname(File.dirname(__FILE__)), 'rake_includes')
+
+
+$platform       ||= ENV['platform']
+$organism       ||= [$organism, ENV['organism'], nil].compact.first 
+$dataset        ||= ENV['dataset']
+
+def process_list
+  list = {}
+  organisms = $organism ? [$organism] : Organism.all(true)
+
+  # Find candidate datasets
+  case
+  when $dataset
+    list[$dataset] = [$dataset]
+  when $platform
+    list[$platform] = [$platform]
+  else
+    organisms.each do |organism|
+      platforms = CustomDS.organism_platforms(organism)
+      platforms.each do |platform|
+        list[platform] = platform
       end
-    }
-  }
-
-end
-
-
-
-
-task 'annotate_Words' do
-  annotations('Words'){|dataset|
-    description = Open.read(CustomDS.path(dataset) + '.description')
-    terms[:dataset] = [dataset] +  description.words
-    Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
-      name = name.strip
-      terms[name] = name.sub(/\[ratio\]/,'').words
-    }
-    terms
-  }
-end
-
-task 'annotate_UMLS' do
-  annotations('Words'){|dataset|
-    description = Open.read(CustomDS.path(dataset) + '.description')
-    terms[:dataset] = Annotations::UMLS::OBA(description).uniq
-    Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
-      name = name.strip
-      terms[name] = Annotations::UMLS::OBA(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).uniq
-    }
-    terms
-  }
-end
-
-
-
-task 'annotate_Polysearch' do
-  require 'rbbt/sources/polysearch'
-  annotations('Polysearch'){|dataset|
-    terms = {}
-    description = Open.read(CustomDS.path(dataset) + '.description')
-    terms[:dataset] = Polysearch::match(description).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
-    Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
-      name = name.strip
-      terms[name] = Polysearch::match(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
-    }
-    terms
-  }
-
-end
-
-task 'annotate_GO' do
-  require 'rbbt/sources/go'
-  options = { :cut_off => $expr_threshold, :fdr => $fdr, :folds => $folds, :do_folds => $do_folds, :nth_genes => $nth_genes}
-  annotations('GO_up', true){|dataset|
-    org = MARQ.platform_organism(dataset)
-    genes = Annotations::GO.get_genes(dataset, options)
-
-    up = {}
-    genes[:up] ||= []
-    genes[:up].collect{|experiment,list|
-      up[experiment] =  goterms(org, list, false, $expr_threshold)
-    }
-    up
-  }
-
-  annotations('GO_down', true){|dataset|
-    org = MARQ.platform_organism(dataset)
-    genes = Annotations::GO.get_genes(dataset, options)
-
-    down = {}
-    genes[:down] ||= []
-    genes[:down].collect{|experiment,list|
-      down[experiment] = goterms(org, list, false, $expr_threshold)
-    }
-    down
-  } 
-  annotations('GOSlim_up', true){|dataset|
-    org = MARQ.platform_organism(dataset)
-    genes = Annotations::GO.get_genes(dataset, options)
-
-    up = {}
-    genes[:up] ||= []
-    genes[:up].collect{|experiment,list|
-      up[experiment] = goterms(org, list, true, $expr_threshold)
-    }
-    up
-  }
-
-  annotations('GOSlim_down', true){|dataset|
-    org = MARQ.platform_organism(dataset)
-    genes = Annotations::GO.get_genes(dataset, options)
-
-    down = {}
-    genes[:down] ||= []
-    genes[:down].collect{|experiment,list|
-      down[experiment] =  goterms(org, list, true, $expr_threshold)
-    }
-    down
-  }
+    end
+  end
+ 
+  return list
 end
 
-task 'default' do
-  Rake::Task['data'].invoke
-  Rake::Task['annotate_Words'].invoke
-  Rake::Task['annotate_UMLS'].invoke
-  Rake::Task['annotate_Polysearch'].invoke
-  Rake::Task['annotate_GO'].invoke
-end

data/install_scripts/GEO/Rakefile

@@ -6,304 +6,63 @@ require 'yaml'
 require 'progress-monitor'
 require 'MARQ/MADB'
 require 'rbbt/sources/organism'
+require File.join(File.dirname(File.dirname(__FILE__)), 'rake_includes')
+
 
 $platform       ||= ENV['platform']
 $organism       ||= [$organism, ENV['organism'], nil].compact.first 
 $dataset        ||= ENV['dataset']
 
-$expr_threshold ||= (ENV['threshold'] || 0.05).to_f
-$folds          ||= (ENV['folds'] || 2.5).to_f
-$nth_genes      ||= (ENV['nth_genes'] || 100).to_i
-
-$force       = [$force, ENV['force'], false].compact.first.to_s == 'true'  
-$tranlations = [$tranlations, ENV['translations'], false].compact.first.to_s == 'true'  
-$series      = [$series, ENV['series'], true].compact.first.to_s == 'true' 
-$update_db   = [$update_db, ENV['update_db'], false].compact.first.to_s == 'true'  
-$skip_db     = [$skip_db, ENV['skip_db'], false].compact.first.to_s == 'true'  
-$fdr         = [$fdr, ENV['fdr'], true].compact.first.to_s == 'true'  
-$do_folds    = [$do_folds, ENV['do_folds'], true].compact.first.to_s == 'true'  
-
+# More global variables in rake_includes file
 
 # Record changes in order to update DB
-$changes = false
-module GEO::Process::R
-  class << self
-    alias_method :GDS_old, :GDS
-    def GDS(*args)
-      $changes = true
-      GDS_old(*args)
-    end
-  
-    alias_method :GSE_old, :GSE
-    def GSE(*args)
-      $changes = true
-      GSE_old(*args)
-    end
-  end
-end
-
 
 def process_list
-  return {GEO::Remote::dataset_platform($dataset) => [$dataset]} if $dataset
-  return {$platform => GEO::Remote::platform_datasets($platform)} if $platform
-  
-  if $organism
-    organisms = [$organism]
-  else
-    organisms = Organism.all(true)
-  end
-
   list = {}
-  organisms.each{|organism|
-    GEO::Remote::organism_platforms(organism).each{|platform|
-      datasets = GEO::Remote::platform_datasets(platform)
-
-      # Platforms with no datasets are skiped, although if they have series
-      # they may be considered later
-      next if datasets.empty?
-
-      if $tranlations || $update_db
-        list[platform] = []
-      else
-        if ! $force
-          datasets.reject!{|dataset| Dir.glob(File.join(GEO.dataset_path(dataset, platform) || "MISSING", '.*')).any? }
-        end
-        list[platform] = datasets
+  organisms = $organism ? [$organism] : Organism.all(true)
+
+  # Find candidate datasets
+  case
+  when $dataset
+    list[GEO::Remote::dataset_platform($dataset)] = [$dataset]
+  when $platform
+    list[$platform] = GEO::Remote::platform_datasets($platform)
+  else
+    organisms.each do |organism|
+      org_platform = GEO::Remote::organism_platforms(organism)
+      org_platform.each do |platform|
+        datasets = GEO::Remote::platform_datasets(platform)
+        list[platform] = datasets if datasets.any?
       end
-    }
-  }
-
-  if $series && ! $translations 
+    end
+  end
+  
+  # Find cadidate series
+  case
+  when $dataset 
+  when $series
     series = Dir.glob('series/*.yaml').collect{|f| File.basename(f).sub(/.yaml/,'')}               
-    series.each{|serie|
+    series.each do |serie|
       platform = GEO::SOFT.GSE(serie)[:platform]
+      next if $platform && $platform != platform
       begin
-        if organisms.include? GEO::SOFT::GPL(platform)[:organism].to_s
+        if platform =~ /_/
+          organism = GEO::SOFT.GPL(platform)[:organism]
+        else
+          organism =  GEO::Remote::platform_organism(platform)
+        end
+        if organism.split(',').select{|org| organisms.include? Organism.name2org(org.strip)}.any?
           list[platform] ||= []
-          list[platform] << serie unless $tranlations || $update_db    
+          list[platform] << serie 
         end
       rescue
         puts "Error process series #{serie} platform #{platform}"
         puts $!.message
         puts $!.backtrace.join("\n")
       end
-    }
-  end
-
-  return list
-end
-
-desc "Analyze datasets"
-task 'data' do 
-
-  platforms_to_save = []
-
-  process_list.each{|platform, datasets|
-
-    begin
-      # Prepare the platform
-      GEO::Process.GPL(platform)
-    rescue
-      puts "Error processing platform #{platform}"
-      puts $!.message
-      puts $!.backtrace.join("\n")
-      next
     end
-
-    $changes = false
-    # Process all datasets
-    datasets.each{|dataset|
-      begin
-        if dataset =~ /GDS/
-          GEO::Process.GDS(dataset, platform)
-        else
-          info = YAML::load(File.open("series/#{ dataset }.yaml"))
-          FileUtils.rm("platforms/#{ info[:platform] }.skip") if File.exist?  "platforms/#{ info[:platform] }.skip"
-          GEO::Process.GSE(dataset, info)
-        end
-
-        # Mark the platform for saving in DB
-      rescue
-        puts "Error processing dataset #{ dataset }"
-        puts $!.message
-        puts $!.backtrace.join("\n")
-      end
-    }
-
-    platforms_to_save << platform if $changes || $update_db
-  }
-
-  platforms_to_save.each{|platform|
-    begin
-      puts "Saving #{platform}"
-      MADB::GEO.saveGPL(platform) 
-    rescue
-      puts "Error saving platform #{ platform }"
-      puts $!.message
-      puts $!.backtrace.join("\n")
-    end
-  }
-end
-
-def annotations(name, cross_platform = false, &block)
-  FileUtils.mkdir_p File.join("annotations", name) unless File.exist? File.join("annotations", name)
-  platforms = Dir.glob('platforms/*.yaml').collect{|f| File.basename(f).sub(/.yaml/,'')}
-  platforms = platforms.sort_by{|platform| MARQ.platform_organism(platform)}.reverse
-
-  if $platform
-    platforms = [$platform]
   end
 
-  Progress.monitor("Annotating with #{ name }")
-  platforms.each{|platform|
-    Progress.monitor("Annotating with #{ name }: platform #{ platform }")
-    GEO.platform_datasets(platform).each do |dataset|
-      begin
-        next if File.exist? File.join("annotations", name, dataset)
-        filename = File.join("annotations", name, dataset)
-        dataset += '_cross_platform' if cross_platform && GEO.has_cross_platform?(nil, platform)
-        next if File.exist? GEO.dataset_path(dataset) + '.skip'
-        terms = block.call(dataset)
-        Open.write(filename, terms.to_yaml)
-      rescue Exception
-        puts $!.message
-      end
-    end
-  }
-  return if $platform
-
-  Progress.monitor("Annotating with #{ name }, series")
-  series = Dir.glob('series/*.yaml').collect{|f| File.basename(f).sub(/.yaml/,'')}
-  series.each{|serie|
-    begin
-      next if File.exist? File.join("annotations", name, serie)
-      filename = File.join("annotations", name, serie)
-      serie += '_cross_platform' if cross_platform && GEO.has_cross_platform?(serie)
-      next if File.exist? GEO.dataset_path(serie) + '.skip'
-      terms = block.call(serie)
-      Open.write(filename, terms.to_yaml)
-    rescue Exception
-      puts $!.message
-    end
-  }
-end
-
-def goterms(org, list, slim, threshold)
-  return [] if list.empty?
-  results = Annotations::GO::Genecodis::Local.analysis(org, list, slim)
-  results.
-    select{|info| info[:s].to_i  > 2 }.
-    select{|info| info[:hyp_c].to_f < threshold }.
-    collect{|info| info[:items]}.collect{|id| GO::id2name(id)}
-end
-
-
-
-task 'annotate_Words' do
-  require 'MARQ/annotations'
-  require 'rbbt/bow/bow'
-  annotations('Words'){|dataset|
-    terms = {}
-    description = Open.read(GEO.dataset_path(dataset) + '.description')
-    terms[:dataset] = [dataset] +  description.words.uniq
-    Open.read(GEO.dataset_path(dataset) + '.experiments').collect{|name|
-      name = name.strip
-      terms[name] = name.sub(/.*?: /,'').sub(/\[ratio\]/,'').words.uniq
-    }
-    terms
-  }
-end
-
-
-task 'annotate_UMLS' do
-  require 'MARQ/annotations'
-  require 'rbbt/util/misc'
-  annotations('UMLS'){|dataset|
-    terms = {}
-    description = Open.read(GEO.dataset_path(dataset) + '.description')
-    terms[:dataset] = Annotations::UMLS::OBA(description).uniq
-    Open.read(GEO.dataset_path(dataset) + '.experiments').collect{|name|
-      name = name.strip
-      terms[name] = Annotations::UMLS::OBA(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).uniq
-    }
-    terms
-  }
-end
-
-
-task 'annotate_Polysearch' do
-  require 'MARQ/annotations'
-  require 'rbbt/util/misc'
-  require 'rbbt/sources/polysearch'
-  annotations('Polysearch'){|dataset|
-    terms = {}
-    description = Open.read(GEO.dataset_path(dataset) + '.description')
-    terms[:dataset] = Polysearch::match(description).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
-    Open.read(GEO.dataset_path(dataset) + '.experiments').collect{|name|
-      name = name.strip
-      terms[name] = Polysearch::match(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
-    }
-    terms
-  }
-
-end
-
-task 'annotate_GO' do
-  require 'MARQ/annotations'
-  require 'rbbt/sources/go'
-  options = { :cut_off => $expr_threshold, :fdr => $fdr, :folds => $folds, :do_folds => $do_folds, :nth_genes => $nth_genes}
-  annotations('GO_up', true){|dataset|
-    org = MARQ.platform_organism(dataset)
-    genes = Annotations::GO.get_genes(dataset, options)
-
-    up = {}
-    genes[:up] ||= []
-    genes[:up].collect{|experiment,list|
-      up[experiment] =  goterms(org, list, false, $expr_threshold)
-    }
-    up
-  }
-
-  annotations('GO_down', true){|dataset|
-    org = MARQ.platform_organism(dataset)
-    genes = Annotations::GO.get_genes(dataset, options)
-
-    down = {}
-    genes[:down] ||= []
-    genes[:down].collect{|experiment,list|
-      down[experiment] = goterms(org, list, false, $expr_threshold)
-    }
-    down
-  } 
-  annotations('GOSlim_up', true){|dataset|
-    org = MARQ.platform_organism(dataset)
-    genes = Annotations::GO.get_genes(dataset, options)
-
-    up = {}
-    genes[:up] ||= []
-    genes[:up].collect{|experiment,list|
-      up[experiment] = goterms(org, list, true, $expr_threshold)
-    }
-    up
-  }
-
-  annotations('GOSlim_down', true){|dataset|
-    org = MARQ.platform_organism(dataset)
-    genes = Annotations::GO.get_genes(dataset, options)
-
-    down = {}
-    genes[:down] ||= []
-    genes[:down].collect{|experiment,list|
-      down[experiment] =  goterms(org, list, true, $expr_threshold)
-    }
-    down
-  }
+  return list
 end
 
-task 'default' do
-  Rake::Task['data'].invoke
-  Rake::Task['annotate_Words'].invoke
-  Rake::Task['annotate_UMLS'].invoke
-  Rake::Task['annotate_Polysearch'].invoke
-  Rake::Task['annotate_GO'].invoke
-end