MARQ 0.0.1

data/README.rdoc

@@ -0,0 +1,29 @@
+= MARQ
+
+Description goes here.
+
+==
+* Install R
+  * Install bioconductor
+* Install mysql
+  * Create database
+    CREATE DATABASE madb;
+    CREATE USER 'marq'@'localhost' IDENTIFIED BY 'marq_password';
+    GRANT ALL PRIVILEGES ON madb.* TO 'marq'@'localhost';
+* MARQ depends on many gems. Some may be problematic to install, in particular
+  rsruby. You may need to get the source and build it yourself instead 
+  of using the gem command. http://github.com/alexgutteridge/rsruby
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2009 Miguel Vazquez. See LICENSE for details.

data/bin/marq_config

@@ -0,0 +1,170 @@
+#!/usr/bin/ruby 
+
+require 'simpleconsole'
+
+def configure
+  config = {}
+  require 'yaml'
+
+  directories = { 
+    'cachedir' => ['cache', "Cache Directory: This directory will hold downloads, from GEO, Arrayexpress, etc."],
+    'datadir' =>  ['data', "Data Directory: This directory will hold the data files produced when processing GEO, Arrayexpress, etc."],
+    'workdir' =>  ['work', "Work Directory: This directory will be used by the Web Server to hold job files."],
+  
+  }
+
+  defaultdir = File.join(ENV['HOME'],'MARQ')
+
+  puts "Please indicate where you wish to place the data directories: "
+  puts
+    
+  directories.each{|name, info|
+    puts info[1]
+    default = File.join(defaultdir, info[0])
+    print "[#{default}]? "
+    input = STDIN.gets
+    input = default unless input =~ /\w/
+    config[name] = input
+  }
+
+  options = {
+    'dbname' => ['madb',"Database Name"],
+    'dbuser' => ['marq',"Database User"],
+    'dbpass' => ['marq_password',"Database Password"],
+    'dbhost' => ['localhost',"Database Host"],
+  }
+
+
+  puts "Please indicate other configuration variables: "
+  puts
+    
+  options.each{|name, info|
+    puts info[1]
+    default = info[0]
+    print "[#{default}]? "
+    input = STDIN.gets
+    input = default unless input =~ /\w/
+    config[name] = input
+  }
+
+  fout = File.open(File.join(ENV['HOME'], '.MARQ'), 'w')
+  fout.write config.to_yaml
+  fout.close
+
+  require 'MARQ'
+
+end
+
+begin
+  require 'MARQ'
+rescue MARQ::NoConfig
+  configure
+  exit
+end
+
+
+$USAGE =<<EOT
+#{__FILE__} <action> [<subaction>] [--force (true|false)] [--update_db (true|false)] [--platform <gpl>] [--series <gse>] [--organism <org>] 
+  actions:
+    * configure:   Set paths for data, cache, and tmp directories
+
+    * install:
+      * GEO: Install management files to process GEO datasets and series
+      * CustomDS: Install management files to process data from other sources
+      * identifiers: Load the identifier equivalences in the database
+
+    * update:
+      * GEO: Process datasets and series from GEO
+      * CustomDS: Process datasets from other sources
+
+                
+                   
+EOT
+
+class Controller < SimpleConsole::Controller
+  params :string => {:p => :platform, :s => :series,  :o => :organism}, :boolean => {:d => :update_db, :f => :force}
+
+
+  def install
+    $organism  = params[:organism] unless params[:organism].nil?
+    @actions = params[:id] || %w(GEO)
+  end
+
+  def update
+    $platform  = params[:platform] unless params[:platform].nil?
+    $series    = params[:series]   unless params[:series].nil?
+    $organism  = params[:organism] unless params[:organism].nil?
+    $update_db = params[:update_db] && params[:update_db] =~ /true|yes|y/i
+    $force     = params[:force]     && params[:force]     =~ /true|yes|y/i
+    @actions   = params[:id] || %w(GEO)
+  end
+
+  def config
+  end
+
+
+
+  def default
+    render :action => :usage
+  end
+
+
+end
+
+
+class View < SimpleConsole::View
+
+  def usage
+    puts $USAGE
+  end
+
+  def config
+    configure
+  end
+
+  def install
+    require 'rbbt/sources/organism'
+    require 'MARQ/ID'
+    case
+    when @actions == 'identifiers'
+      if $organism
+        organisms = [$organism]
+      else
+        organisms = Organism.all
+      end
+      organisms.each{|org|
+        ID.DB_load(org)
+      }
+    else
+      require 'rake'
+      load File.join(MARQ.rootdir, 'tasks/install.rake')
+
+      @actions = [@actions] if @actions === String
+      @actions.each{|action|
+        puts "Installing #{ action }"
+        Rake::Task[action].invoke
+      }
+    end
+  end
+
+  def update
+    install
+
+    require 'rake'
+    @actions = [@actions] if @actions === String
+
+    @actions.each{|action|
+      Thread.new{
+        puts "Updating #{action}"
+        FileUtils.cd File.join(MARQ.datadir, action)
+        load File.join(MARQ.datadir, action, 'Rakefile')
+        Rake::Task['default'].invoke
+      }.join
+    }
+  end
+
+
+end
+
+SimpleConsole::Application.run(ARGV, Controller, View)
+

data/install_scripts/CustomDS/Rakefile

@@ -0,0 +1,223 @@
+require 'MARQ'
+require 'MARQ/CustomDS'
+require 'MARQ/MADB'
+require 'MARQ/annotations'
+require 'progress-monitor'
+
+$org            ||= ENV['organism']
+
+$expr_threshold ||= (ENV['threshold'] || 0.05).to_f
+$folds          ||= (ENV['folds'] || 2.5).to_f
+$nth_genes      ||= (ENV['nth_genes'] || 100).to_i
+
+$force       = [$force, ENV['force'], false].         reject{|e| e.nil?}.first 
+$update_db   = [$update_db, ENV['update_db'], true].  reject{|e| e.nil?}.first 
+$fdr         = [$fdr, ENV['fdr'], true].              reject{|e| e.nil?}.first 
+$do_folds    = [$do_folds, ENV['do_folds'], true].    reject{|e| e.nil?}.first 
+
+
+
+
+task 'data' do
+  organisms = Dir.glob(CustomDS::customdir + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - ["annotations"]
+  
+  organisms.each{|org|
+    datasets = Dir.glob(File.join(CustomDS::customdir, org) + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)}
+    if $force
+      missing = datasets
+    else
+      missing = datasets - CustomDS::datasets(org)
+    end
+    missing.each{|d|
+      CustomDS::process(d)
+      MADB::CustomDS::save(d)
+    }
+  }
+end
+
+
+def annotations(name, cross_platform = false, &block)
+  FileUtils.mkdir_p File.join("annotations", name) unless File.exist? File.join("annotations", name)
+  organisms = Dir.glob(CustomDS::customdir + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - ["annotations"]
+
+  Progress.monitor("Annotating with #{ name }")
+  organisms.each{|org|
+    Progress.monitor("Annotating organism #{ org }")
+    CustomDS::datasets(org).each{|dataset|
+      begin
+        next if File.exist? File.join("annotations", name, dataset)
+        filename = File.join("annotations", name, dataset)
+        dataset += '_cross_platform' if cross_platform && GEO.has_cross_platform?(dataset)
+        next if File.exist? CustomDS.path(dataset) + '.skip'
+        terms = block.call(dataset)
+        Open.write(filename, terms.to_yaml)
+      rescue Exception
+        puts $!.message
+      end
+    }
+  }
+end
+
+def goterms(org, list, slim, threshold)
+  return [] if list.empty?
+  results = Annotations::GO::Genecodis::Local.analysis(org, list, slim)
+  results.
+    select{|info| info[:s].to_i  > 2 }.
+    select{|info| info[:hyp_c].to_f < threshold }.
+    collect{|info| info[:items]}.collect{|id| GO::id2name(id)}
+end
+
+
+task 'annotate_Words' do
+  FileUtils.mkdir_p "annotations/Words"
+  require 'rbbt/bow/bow'
+  organisms = Dir.glob(CustomDS::customdir + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - ["annotations"]
+  organisms.each{|org|
+    CustomDS::datasets(org).each{|dataset|
+      begin
+        next if File.exist? "annotations/Words/#{ dataset }"
+        next if File.exist? CustomDS.path(dataset) + '.skip'
+
+        terms = {}
+        description = Open.read(CustomDS.path(dataset) + '.description')
+        terms[:dataset] = [dataset] +  description.words
+        Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
+          name = name.strip
+          terms[name] = name.sub(/\[ratio\]/,'').words
+        }
+        
+        Open.write("annotations/Words/#{ dataset }", terms.to_yaml)
+
+      rescue Exception
+        puts $!.message
+      end
+    }
+ }
+end
+
+task 'annotate_UMLS' do
+  FileUtils.mkdir_p "annotations/UMLS"
+  organisms = Dir.glob(CustomDS::customdir + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - ["annotations"]
+  organisms.each{|org|
+    CustomDS::datasets(org).each{|dataset|
+      begin
+        next if File.exist? "annotations/UMLS/#{ dataset }"
+        next if File.exist? CustomDS.path(dataset) + '.skip'
+        terms = {}
+        description = Open.read(CustomDS.path(dataset) + '.description')
+        terms[:dataset] = Annotations::UMLS::OBA(description)
+        Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
+          name = name.strip
+          terms[name] = Annotations::UMLS::OBA(name)
+        }
+        Open.write("annotations/UMLS/#{ dataset }", terms.to_yaml)
+      rescue Exception
+        puts $!.message
+      end
+    }
+  }
+
+end
+
+
+
+
+task 'annotate_Words' do
+  annotations('Words'){|dataset|
+    description = Open.read(CustomDS.path(dataset) + '.description')
+    terms[:dataset] = [dataset] +  description.words
+    Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = name.sub(/\[ratio\]/,'').words
+    }
+    terms
+  }
+end
+
+task 'annotate_UMLS' do
+  annotations('Words'){|dataset|
+    description = Open.read(CustomDS.path(dataset) + '.description')
+    terms[:dataset] = Annotations::UMLS::OBA(description).uniq
+    Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = Annotations::UMLS::OBA(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).uniq
+    }
+    terms
+  }
+end
+
+
+
+task 'annotate_Polysearch' do
+  require 'rbbt/sources/polysearch'
+  annotations('Polysearch'){|dataset|
+    terms = {}
+    description = Open.read(CustomDS.path(dataset) + '.description')
+    terms[:dataset] = Polysearch::match(description).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
+    Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = Polysearch::match(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
+    }
+    terms
+  }
+
+end
+
+task 'annotate_GO' do
+  require 'rbbt/sources/go'
+  options = { :cut_off => $expr_threshold, :fdr => $fdr, :folds => $folds, :do_folds => $do_folds, :nth_genes => $nth_genes}
+  annotations('GO_up', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    up = {}
+    genes[:up] ||= []
+    genes[:up].collect{|experiment,list|
+      up[experiment] =  goterms(org, list, false, $expr_threshold)
+    }
+    up
+  }
+
+  annotations('GO_down', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    down = {}
+    genes[:down] ||= []
+    genes[:down].collect{|experiment,list|
+      down[experiment] = goterms(org, list, false, $expr_threshold)
+    }
+    down
+  } 
+  annotations('GOSlim_up', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    up = {}
+    genes[:up] ||= []
+    genes[:up].collect{|experiment,list|
+      up[experiment] = goterms(org, list, true, $expr_threshold)
+    }
+    up
+  }
+
+  annotations('GOSlim_down', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    down = {}
+    genes[:down] ||= []
+    genes[:down].collect{|experiment,list|
+      down[experiment] =  goterms(org, list, true, $expr_threshold)
+    }
+    down
+  }
+end
+
+task 'default' do
+  Rake::Task['data'].invoke
+  Rake::Task['annotate_Words'].invoke
+  Rake::Task['annotate_UMLS'].invoke
+  Rake::Task['annotate_Polysearch'].invoke
+  Rake::Task['annotate_GO'].invoke
+end

data/install_scripts/GEO/Rakefile

@@ -0,0 +1,258 @@
+require 'MARQ'
+require 'MARQ/GEO'
+require 'MARQ/ID'
+require 'yaml'
+require 'progress-monitor'
+require 'MARQ/MADB'
+require 'MARQ/annotations'
+
+$platform       ||= ENV['platform']
+$org            ||= [$organism, ENV['organism'], nil].reject{|e| e.nil?}.first 
+$series         ||= ENV['series']
+
+$expr_threshold ||= (ENV['threshold'] || 0.05).to_f
+$folds          ||= (ENV['folds'] || 2.5).to_f
+$nth_genes      ||= (ENV['nth_genes'] || 100).to_i
+
+$force       = [$force, ENV['force'], false].         reject{|e| e.nil?}.first 
+$update_db   = [$update_db, ENV['update_db'], true].  reject{|e| e.nil?}.first 
+$fdr         = [$fdr, ENV['fdr'], true].              reject{|e| e.nil?}.first 
+$do_folds    = [$do_folds, ENV['do_folds'], true].    reject{|e| e.nil?}.first 
+
+
+# Record changes in order to update DB
+$changes = false
+module GEO
+  class << self
+    alias_method :get_GDS_old, :get_GDS
+    def get_GDS(*args)
+      $changes = true
+      get_GDS_old(*args)
+    end
+  
+    alias_method :get_GSE_old, :get_GSE
+    def get_GSE(*args)
+      $changes = true
+      get_GSE_old(*args)
+    end
+  end
+end
+
+
+def process_platform(platform, update_db = false)
+  begin
+    $changes = false
+    GEO.process_platform(platform)
+    GEO.process_platform_datasets(platform, $force && !update_db)
+    MADB::GEO.saveGPL(platform) if update_db && ($changes || $force)
+  rescue
+    puts $!.message
+  end
+end
+
+def process_serie(serie, update_db = false)
+  begin
+    $changes = false
+    info = YAML::load(File.open("series/#{ serie }.yaml"))
+
+    FileUtils.rm("platforms/#{ info[:platform] }.skip") if File.exist?  "platforms/#{ info[:platform] }.skip"
+    GEO.process_platform(info[:platform]) if !File.exist? info[:platform]
+
+    GEO.process_GSE(serie, info)
+    MADB::GEO.saveGPL(info[:platform]) if info[:platform] && update_db
+    info[:platform]
+  rescue
+    puts $!.message
+  end
+end
+
+def annotations(name, cross_platform = false, &block)
+  FileUtils.mkdir_p File.join("annotations", name) unless File.exist? File.join("annotations", name)
+  platforms = Dir.glob('platforms/*.yaml').collect{|f| File.basename(f).sub(/.yaml/,'')}
+  platforms = platforms.sort_by{|platform| MARQ.platform_organism(platform)}.reverse
+
+  if $platform
+    platforms = [$platform]
+  end
+
+  Progress.monitor("Annotating with #{ name }")
+  platforms.each{|platform|
+    Progress.monitor("Annotating with #{ name }: platform #{ platform }")
+    GEO.platform_datasets(platform).each do |dataset|
+      begin
+        next if File.exist? File.join("annotations", name, dataset)
+        filename = File.join("annotations", name, dataset)
+        dataset += '_cross_platform' if cross_platform && GEO.has_cross_platform?(nil, platform)
+        next if File.exist? GEO.dataset_path(dataset) + '.skip'
+        terms = block.call(dataset)
+        Open.write(filename, terms.to_yaml)
+      rescue Exception
+        puts $!.message
+      end
+    end
+  }
+  return if $platform
+  
+  Progress.monitor("Annotating with #{ name }, series")
+  series = Dir.glob('series/*.yaml').collect{|f| File.basename(f).sub(/.yaml/,'')}
+  series.each{|serie|
+    begin
+      next if File.exist? File.join("annotations", name, serie)
+      filename = File.join("annotations", name, serie)
+      serie += '_cross_platform' if cross_platform && GEO.has_cross_platform?(serie)
+      next if File.exist? GEO.dataset_path(serie) + '.skip'
+      terms = block.call(serie)
+      Open.write(filename, terms.to_yaml)
+    rescue Exception
+      puts $!.message
+    end
+  }
+end
+
+def goterms(org, list, slim, threshold)
+  return [] if list.empty?
+  results = Annotations::GO::Genecodis::Local.analysis(org, list, slim)
+  results.
+    select{|info| info[:s].to_i  > 2 }.
+    select{|info| info[:hyp_c].to_f < threshold }.
+    collect{|info| info[:items]}.collect{|id| GO::id2name(id)}
+end
+
+
+task 'data' do
+  if $platform
+    process_platform($platform, $update_db)
+  elsif $series
+    process_serie($series, $update_db)
+  elsif $org
+    org = $org
+    platforms = GEO::Eutils.organism_platforms(org).select{|platform|  GEO::Eutils.GPL_datasets(platform).any?}
+    Progress.monitor("Platforms for #{Organism.name(org)}")
+    platforms.each{|platform|
+      puts "Platform #{ platform }"
+      process_platform(platform,$update_db)
+    }
+  else
+    # Series
+    series = Dir.glob('series/*.yaml').collect{|f| File.basename(f).sub(/.yaml/,'')}
+    platform2save = []
+    series.each{|serie|
+      platform  = process_serie(serie, false)
+      platform2save << platform if $update_db && ($changes || $force)
+    }
+
+    platform2save.uniq.each{|platform| MADB::GEO.saveGPL(platform) }
+
+    # Platforms
+    for org in Organism.all
+      platforms = GEO::Eutils.organism_platforms(org).select{|platform|  GEO::Eutils.GPL_datasets(platform).any?}
+      Progress.monitor("Platforms for #{Organism.name(org)}")
+      platforms.each{|platform|
+        puts "Platform #{ platform }"
+        process_platform(platform,$update_db)
+      }
+    end
+  end
+end
+
+task 'annotate_Words' do
+  annotations('Words'){|dataset|
+    terms = {}
+    description = Open.read(GEO.dataset_path(dataset) + '.description')
+    terms[:dataset] = [dataset] +  description.words.uniq
+    Open.read(GEO.dataset_path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = name.sub(/.*?: /,'').sub(/\[ratio\]/,'').words.uniq
+    }
+    terms
+  }
+end
+
+
+task 'annotate_UMLS' do
+  annotations('UMLS'){|dataset|
+    terms = {}
+    description = Open.read(GEO.dataset_path(dataset) + '.description')
+    terms[:dataset] = Annotations::UMLS::OBA(description).uniq
+    Open.read(GEO.dataset_path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = Annotations::UMLS::OBA(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).uniq
+    }
+    terms
+  }
+end
+
+
+task 'annotate_Polysearch' do
+  require 'rbbt/sources/polysearch'
+  annotations('Polysearch'){|dataset|
+    terms = {}
+    description = Open.read(GEO.dataset_path(dataset) + '.description')
+    terms[:dataset] = Polysearch::match(description).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
+    Open.read(GEO.dataset_path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = Polysearch::match(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
+    }
+    terms
+  }
+
+end
+
+task 'annotate_GO' do
+  require 'rbbt/sources/go'
+  options = { :cut_off => $expr_threshold, :fdr => $fdr, :folds => $folds, :do_folds => $do_folds, :nth_genes => $nth_genes}
+  annotations('GO_up', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    up = {}
+    genes[:up] ||= []
+    genes[:up].collect{|experiment,list|
+      up[experiment] =  goterms(org, list, false, $expr_threshold)
+    }
+    up
+  }
+
+  annotations('GO_down', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    down = {}
+    genes[:down] ||= []
+    genes[:down].collect{|experiment,list|
+      down[experiment] = goterms(org, list, false, $expr_threshold)
+    }
+    down
+  } 
+  annotations('GOSlim_up', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    up = {}
+    genes[:up] ||= []
+    genes[:up].collect{|experiment,list|
+      up[experiment] = goterms(org, list, true, $expr_threshold)
+    }
+    up
+  }
+
+  annotations('GOSlim_down', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    down = {}
+    genes[:down] ||= []
+    genes[:down].collect{|experiment,list|
+      down[experiment] =  goterms(org, list, true, $expr_threshold)
+    }
+    down
+  }
+end
+
+task 'default' do
+  Rake::Task['data'].invoke
+  Rake::Task['annotate_Words'].invoke
+  Rake::Task['annotate_UMLS'].invoke
+  Rake::Task['annotate_Polysearch'].invoke
+  Rake::Task['annotate_GO'].invoke
+end