rbbt-marq 1.0.0

data/install_scripts/GEO/Rakefile

@@ -0,0 +1,272 @@
+require 'MARQ'
+require 'MARQ/main'
+require 'MARQ/GEO'
+require 'MARQ/ID'
+require 'yaml'
+require 'progress-monitor'
+require 'MARQ/MADB'
+
+$platform       ||= ENV['platform']
+$org            ||= [$organism, ENV['organism'], nil].reject{|e| e.nil?}.first 
+$series         ||= ENV['series']
+
+$expr_threshold ||= (ENV['threshold'] || 0.05).to_f
+$folds          ||= (ENV['folds'] || 2.5).to_f
+$nth_genes      ||= (ENV['nth_genes'] || 100).to_i
+
+$force       = [$force, ENV['force'], false].         reject{|e| e.nil?}.first 
+$update_db   = [$update_db, ENV['update_db'], true].  reject{|e| e.nil?}.first 
+$fdr         = [$fdr, ENV['fdr'], true].              reject{|e| e.nil?}.first 
+$do_folds    = [$do_folds, ENV['do_folds'], true].    reject{|e| e.nil?}.first 
+
+
+# Record changes in order to update DB
+$changes = false
+module GEO
+  class << self
+    alias_method :get_GDS_old, :get_GDS
+    def get_GDS(*args)
+      $changes = true
+      get_GDS_old(*args)
+    end
+  
+    alias_method :get_GSE_old, :get_GSE
+    def get_GSE(*args)
+      $changes = true
+      get_GSE_old(*args)
+    end
+  end
+end
+
+
+def process_platform(platform, update_db = false)
+  begin
+    $changes = false
+    GEO.process_platform(platform)
+    GEO.process_platform_datasets(platform, $force && !update_db)
+    if update_db && ($changes || $force)
+      puts "Saving #{platform}"       
+      MADB::GEO.saveGPL(platform) if update_db && ($changes || $force)
+    end
+  rescue
+    puts $!.message
+  end
+end
+
+def process_serie(serie, update_db = false)
+  begin
+    $changes = false
+    info = YAML::load(File.open("series/#{ serie }.yaml"))
+
+    FileUtils.rm("platforms/#{ info[:platform] }.skip") if File.exist?  "platforms/#{ info[:platform] }.skip"
+    GEO.process_platform(info[:platform]) if !File.exist? info[:platform]
+
+    GEO.process_GSE(serie, info)
+    if info[:platform] && update_db
+      puts "Saving #{info[:platform]}"
+      MADB::GEO.saveGPL(info[:platform]) if info[:platform] && update_db
+    end
+    info[:platform]
+  rescue
+    puts $!.message
+  end
+end
+
+def annotations(name, cross_platform = false, &block)
+  FileUtils.mkdir_p File.join("annotations", name) unless File.exist? File.join("annotations", name)
+  platforms = Dir.glob('platforms/*.yaml').collect{|f| File.basename(f).sub(/.yaml/,'')}
+  platforms = platforms.sort_by{|platform| MARQ.platform_organism(platform)}.reverse
+
+  if $platform
+    platforms = [$platform]
+  end
+
+  Progress.monitor("Annotating with #{ name }")
+  platforms.each{|platform|
+    Progress.monitor("Annotating with #{ name }: platform #{ platform }")
+    GEO.platform_datasets(platform).each do |dataset|
+      begin
+        next if File.exist? File.join("annotations", name, dataset)
+        filename = File.join("annotations", name, dataset)
+        dataset += '_cross_platform' if cross_platform && GEO.has_cross_platform?(nil, platform)
+        next if File.exist? GEO.dataset_path(dataset) + '.skip'
+        terms = block.call(dataset)
+        Open.write(filename, terms.to_yaml)
+      rescue Exception
+        puts $!.message
+      end
+    end
+  }
+  return if $platform
+  
+  Progress.monitor("Annotating with #{ name }, series")
+  series = Dir.glob('series/*.yaml').collect{|f| File.basename(f).sub(/.yaml/,'')}
+  series.each{|serie|
+    begin
+      next if File.exist? File.join("annotations", name, serie)
+      filename = File.join("annotations", name, serie)
+      serie += '_cross_platform' if cross_platform && GEO.has_cross_platform?(serie)
+      next if File.exist? GEO.dataset_path(serie) + '.skip'
+      terms = block.call(serie)
+      Open.write(filename, terms.to_yaml)
+    rescue Exception
+      puts $!.message
+    end
+  }
+end
+
+def goterms(org, list, slim, threshold)
+  return [] if list.empty?
+  results = Annotations::GO::Genecodis::Local.analysis(org, list, slim)
+  results.
+    select{|info| info[:s].to_i  > 2 }.
+    select{|info| info[:hyp_c].to_f < threshold }.
+    collect{|info| info[:items]}.collect{|id| GO::id2name(id)}
+end
+
+
+task 'data' do
+  if $platform
+    process_platform($platform, $update_db)
+  elsif $series
+    process_serie($series, $update_db)
+  elsif $org
+    org = $org
+    platforms = GEO::Eutils.organism_platforms(org).select{|platform|  GEO::Eutils.GPL_datasets(platform).any?}
+    Progress.monitor("Platforms for #{Organism.name(org)}")
+    platforms.each{|platform|
+      puts "Platform #{ platform }"
+      process_platform(platform,$update_db)
+    }
+  else
+    # Series
+    series = Dir.glob('series/*.yaml').collect{|f| File.basename(f).sub(/.yaml/,'')}
+    platform2save = []
+    series.each{|serie|
+      platform  = process_serie(serie, false)
+      platform2save << platform if $update_db && ($changes || $force)
+    }
+
+    Progress.monitor("Saving #{platform2save.uniq.length} platforms: ")
+    platform2save.uniq.each{|platform| MADB::GEO.saveGPL(platform) }
+
+    # Platforms
+    for org in Organism.all
+      platforms = GEO::Eutils.organism_platforms(org).select{|platform|  GEO::Eutils.GPL_datasets(platform).any?}
+      Progress.monitor("Platforms for #{Organism.name(org)}")
+      platforms.each{|platform|
+        puts "Platform #{ platform }"
+        process_platform(platform,$update_db)
+      }
+    end
+  end
+end
+
+task 'annotate_Words' do
+  require 'MARQ/annotations'
+  require 'rbbt/util/misc'
+  annotations('Words'){|dataset|
+    terms = {}
+    description = Open.read(GEO.dataset_path(dataset) + '.description')
+    terms[:dataset] = [dataset] +  description.words.uniq
+    Open.read(GEO.dataset_path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = name.sub(/.*?: /,'').sub(/\[ratio\]/,'').words.uniq
+    }
+    terms
+  }
+end
+
+
+task 'annotate_UMLS' do
+  require 'MARQ/annotations'
+  require 'rbbt/util/misc'
+  annotations('UMLS'){|dataset|
+    terms = {}
+    description = Open.read(GEO.dataset_path(dataset) + '.description')
+    terms[:dataset] = Annotations::UMLS::OBA(description).uniq
+    Open.read(GEO.dataset_path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = Annotations::UMLS::OBA(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).uniq
+    }
+    terms
+  }
+end
+
+
+task 'annotate_Polysearch' do
+  require 'MARQ/annotations'
+  require 'rbbt/util/misc'
+  require 'rbbt/sources/polysearch'
+  annotations('Polysearch'){|dataset|
+    terms = {}
+    description = Open.read(GEO.dataset_path(dataset) + '.description')
+    terms[:dataset] = Polysearch::match(description).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
+    Open.read(GEO.dataset_path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = Polysearch::match(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
+    }
+    terms
+  }
+
+end
+
+task 'annotate_GO' do
+  require 'MARQ/annotations'
+  require 'rbbt/sources/go'
+  options = { :cut_off => $expr_threshold, :fdr => $fdr, :folds => $folds, :do_folds => $do_folds, :nth_genes => $nth_genes}
+  annotations('GO_up', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    up = {}
+    genes[:up] ||= []
+    genes[:up].collect{|experiment,list|
+      up[experiment] =  goterms(org, list, false, $expr_threshold)
+    }
+    up
+  }
+
+  annotations('GO_down', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    down = {}
+    genes[:down] ||= []
+    genes[:down].collect{|experiment,list|
+      down[experiment] = goterms(org, list, false, $expr_threshold)
+    }
+    down
+  } 
+  annotations('GOSlim_up', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    up = {}
+    genes[:up] ||= []
+    genes[:up].collect{|experiment,list|
+      up[experiment] = goterms(org, list, true, $expr_threshold)
+    }
+    up
+  }
+
+  annotations('GOSlim_down', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    down = {}
+    genes[:down] ||= []
+    genes[:down].collect{|experiment,list|
+      down[experiment] =  goterms(org, list, true, $expr_threshold)
+    }
+    down
+  }
+end
+
+task 'default' do
+  Rake::Task['data'].invoke
+  Rake::Task['annotate_Words'].invoke
+  Rake::Task['annotate_UMLS'].invoke
+  Rake::Task['annotate_Polysearch'].invoke
+  Rake::Task['annotate_GO'].invoke
+end

data/install_scripts/GEO/platforms/GPL100.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: Affymetrix GeneChip Human 35K SubC Array Hu35k-C
+:BioMart_ID: 
+:other_ID_field: 
+- 10
+- ENTREZ_GENE_ID
+:organism: human

data/install_scripts/GEO/platforms/GPL1002.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: Rat 5K cDNA microarray
+:BioMart_ID: 
+:other_ID_field: 
+- 3
+- SPOT_ID
+:organism: rgd

data/install_scripts/GEO/platforms/GPL1007.yaml

@@ -0,0 +1,7 @@
+--- 
+:other_ID_field: 
+- 5
+- ORF
+:title: 70 bp yeast oligo array
+:organism: sgd
+:BioMart_ID: 

data/install_scripts/GEO/platforms/GPL101.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: Affymetrix GeneChip Human 35K SubD Array Hu35k-D
+:BioMart_ID: 
+:other_ID_field: 
+- 10
+- ENTREZ_GENE_ID
+:organism: human

data/install_scripts/GEO/platforms/GPL1010.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: "MPSS [DpnII: Mus musculus] signature list\r"
+:BioMart_ID: 
+:other_ID_field: 
+- 4
+- SPOT_ID
+:organism: mgi

data/install_scripts/GEO/platforms/GPL1073.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: GNF1M
+:BioMart_ID: 
+:other_ID_field: 
+- 4
+- CLONE_ID
+:organism: mgi

data/install_scripts/GEO/platforms/GPL1074.yaml

@@ -0,0 +1,4 @@
+--- 
+:title: GNF1H
+:BioMart_ID: AFFY HG U133A
+:organism: human

data/install_scripts/GEO/platforms/GPL1090.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: Human KUGI14K
+:BioMart_ID: 
+:other_ID_field: 
+- 3
+- PT_ACC
+:organism: human

data/install_scripts/GEO/platforms/GPL1104.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: Mouse NIA 15K Custom
+:BioMart_ID: 
+:other_ID_field: 
+- 3
+- MESH
+:organism: mgi

data/install_scripts/GEO/platforms/GPL118.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: FH_yeast_cDNA
+:BioMart_ID: 
+:other_ID_field: 
+- 2
+- ORF
+:organism: sgd

data/install_scripts/GEO/platforms/GPL1205.yaml

@@ -0,0 +1,4 @@
+--- 
+:title: Mouse 27K  array (AECOM)
+:BioMart_ID: 
+:organism: mgi

data/install_scripts/GEO/platforms/GPL1211.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: NIA MGC, Mammalian Genome Collection
+:BioMart_ID: 
+:other_ID_field: 
+- 4
+- GENE_SYMBOL
+:organism: human

data/install_scripts/GEO/platforms/GPL1213.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: Human Transporter OligoChip
+:BioMart_ID: 
+:other_ID_field: 
+- 2
+- GI
+:organism: human

data/install_scripts/GEO/platforms/GPL1219.yaml

@@ -0,0 +1,4 @@
+--- 
+:title: Amersham Biosciences CodeLink UniSet Mouse I Bioarray
+:BioMart_ID: 
+:organism: mgi

data/install_scripts/GEO/platforms/GPL1223.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: Arcturus 22k human oligonucleotide microarray
+:BioMart_ID: 
+:other_ID_field: 
+- 5
+- GB_ACC
+:organism: human

data/install_scripts/GEO/platforms/GPL1226.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: Mouse 13.4K long oligonucleotide array (Operon) 9.10.2002v2
+:BioMart_ID: 
+:other_ID_field: 
+- 4
+- CONTROL
+:organism: mgi

data/install_scripts/GEO/platforms/GPL1229.yaml

@@ -0,0 +1,7 @@
+--- 
+:other_ID_field: 
+- 2
+- ORF
+:title: Hughes Lab Saccharomyces cerevisiae cDNA ORF array
+:organism: sgd
+:BioMart_ID: 

data/install_scripts/GEO/platforms/GPL1230.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: SMCmou8400A
+:BioMart_ID: 
+:other_ID_field: 
+- 6
+- SPOT_ID
+:organism: mgi

data/install_scripts/GEO/platforms/GPL1231.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: SMCmou6600Av1
+:BioMart_ID: 
+:other_ID_field: 
+- 6
+- SPOT_ID
+:organism: mgi

data/install_scripts/GEO/platforms/GPL1232.yaml

@@ -0,0 +1,7 @@
+--- 
+:title: SMCmou6600Av2
+:BioMart_ID: 
+:other_ID_field: 
+- 6
+- SPOT_ID
+:organism: mgi