rbbt-marq 1.0.0

data/README.rdoc

@@ -0,0 +1,58 @@
+= MARQ
+
+MicroArray Rank Query (MARQ) is an application used to retrieve microarray
+experiments that are similar to a given query based on a rank statistic. This
+package includes the tools to build the microarray database, a SOAP web server 
+to perform the queries and a merb application that offers a complete visual
+interface to the system with several additional features.
+
+== Installation
+
+Install the gem and all the dependencies. This gem depends on several others,
+including Rbbt. Rbbt offers an API with the gene identifier translation used in
+MARQ. To install MARQ follow this recipie:
+
+* Install Rbbt
+
+    rbbt_config configure
+    rbbt_config install identifiers
+    rbbt_config organisms
+    # Choose the organisms you are interested in, and take the keyword. Like sgd for yeast
+    rbbt_config update organisms -o sgd
+
+* Install R
+  * Install bioconductor and GEOquery
+      source("http://bioconductor.org/biocLite.R")
+      biocLite()
+      biocLite('GEOquery')
+
+* Install mysql
+  * Create database
+      CREATE DATABASE madb;
+      CREATE USER 'marq'@'localhost' IDENTIFIED BY 'marq_password';
+      GRANT ALL PRIVILEGES ON madb.* TO 'marq'@'localhost';
+
+* Install MARQ
+    marq_config config
+    marq_config install identifiers
+    marq_config update GEO
+
+* MARQ depends on many gems. Some may be problematic to install, in particular
+  rsruby. You may need to get the source and build it yourself instead 
+  of using the gem command. http://github.com/alexgutteridge/rsruby. Also RubyInline
+  does not work in ruby1.9 so its require has been delayed to just when its about
+  to be used.
+
+== Note on Patches/Pull Requests
+ 
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2009 Miguel Vazquez. See LICENSE for details.

data/bin/marq_config

@@ -0,0 +1,209 @@
+#!/usr/bin/ruby 
+
+require 'simpleconsole'
+
+def configure
+  config = {}
+  require 'yaml'
+
+  directories = { 
+    'cachedir' => ['cache', "Cache Directory: This directory will hold downloads, from GEO, Arrayexpress, etc."],
+    'datadir' =>  ['data', "Data Directory: This directory will hold the data files produced when processing GEO, Arrayexpress, etc."],
+    'workdir' =>  ['work', "Work Directory: This directory will be used by the Web Server to hold job files."],
+  
+  }
+
+  defaultdir = File.join(ENV['HOME'],'MARQ')
+
+  puts "Please indicate where you wish to place the data directories: "
+  puts
+    
+  directories.each{|name, info|
+    puts info[1]
+    default = File.join(defaultdir, info[0])
+    print "[#{default}]? "
+    input = STDIN.gets
+    input = default unless input =~ /\w/
+    config[name] = input
+  }
+
+  options = {
+    'dbname' => ['madb',"Database Name"],
+    'dbuser' => ['marq',"Database User"],
+    'dbpass' => ['marq_password',"Database Password"],
+    'dbhost' => ['localhost',"Database Host"],
+  }
+
+
+  puts "Please indicate other configuration variables: "
+  puts
+    
+  options.each{|name, info|
+    puts info[1]
+    default = info[0]
+    print "[#{default}]? "
+    input = STDIN.gets
+    input = default unless input =~ /\w/
+    config[name] = input
+  }
+
+  fout = File.open(File.join(ENV['HOME'], '.MARQ'), 'w')
+  fout.write config.to_yaml
+  fout.close
+
+  require 'MARQ'
+
+end
+
+begin
+  require 'MARQ'
+rescue 
+  puts $!.message
+  configure
+  exit
+end
+
+
+$USAGE =<<EOT
+#{__FILE__} <action> [<subaction>] [--force (true|false)] [--update_db (true|false)] [--platform <gpl>] [--series <gse>] [--organism <org>] [--port <number>] [--host <name>]
+  actions:
+    * config:   Set paths for data, cache, and tmp directories
+
+    * install:
+      * GEO: Install management files to process GEO datasets and series
+      * CustomDS: Install management files to process data from other sources
+      * identifiers: Load the identifier equivalences in the database
+
+    * update:
+      * GEO: Process datasets and series from GEO
+      * CustomDS: Process datasets from other sources
+
+    * init:
+      * webservice: Start the SOAP Web Service
+      * www: Start the merb server
+                
+                   
+EOT
+
+class Controller < SimpleConsole::Controller
+  params :string => {:p => :platform, :s => :series,  :o => :organism}, :boolean => {:d => :update_db, :f => :force}, :integer => {:p => :port}
+
+
+  def install
+    $organism  = params[:organism] unless params[:organism].nil?
+    @actions = params[:id] || %w(GEO)
+  end
+
+  def update
+    $platform  = params[:platform] unless params[:platform].nil?
+    $series    = params[:series]   unless params[:series].nil?
+    $organism  = params[:organism] unless params[:organism].nil?
+    $update_db = params[:update_db] && params[:update_db] =~ /true|yes|y/i
+    $force     = params[:force]     && params[:force]     =~ /true|yes|y/i
+    @actions   = params[:id] || %w(GEO)
+  end
+
+  def config
+  end
+
+
+
+  def default
+    render :action => :usage
+  end
+
+  def init
+    $host   = [params[:host], `hostname`.chomp.strip + '.' +  `hostname -d`.chomp.strip].compact.first
+    $port   = params[:port]
+    @action = params[:id]
+  end
+
+
+end
+
+
+class View < SimpleConsole::View
+
+  def init
+    case @action
+    when 'webservice'
+      require 'MARQWS'
+      $port   = [$port, 8282].compact.first
+      FileUtils.cd File.join(MARQ.rootdir, 'webservice')
+
+      puts "Starting Server in #{ $host }:#{ $port }"
+      server = MARQWS.new("MARQ", "MARQ Web Server", $host, $port, File.join(MARQ.workdir,'webservice', 'jobs'))
+
+      FileUtils.mkdir_p File.join(MARQ.workdir, '/webservice', 'wsdl/') unless File.exist? File.join(MARQ.workdir, '/webservice', 'wsdl/')
+      File.open(File.join(MARQ.workdir, '/webservice', 'wsdl', 'MARQWS.wsdl'),'w'){|file| file.write server.wsdl}
+
+      trap('INT') { server.shutdown }
+      server.start
+    when 'www'
+      require 'merb-core'
+      $port   = [$port, 8281].compact.first
+      
+      FileUtils.rm Dir.glob(File.join(MARQ.workdir, 'merb','cache','*.lock'))
+
+      puts(File.join(MARQ.rootdir, 'merb'))
+      Dir.chdir(File.join(MARQ.rootdir, 'merb'))
+      puts Dir.pwd
+
+      Merb.start("-a mongrel -e production -p #{ $port } -h #{ $host }".split)
+    end
+  end
+
+  def usage
+    puts $USAGE
+  end
+
+  def config
+    configure
+  end
+
+  def install
+    require 'rbbt/sources/organism'
+    require 'MARQ/ID'
+    case
+    when @actions == 'identifiers'
+      if $organism
+        organisms = [$organism]
+      else
+        organisms = Organism.all
+      end
+      organisms.each{|org|
+        ID.DB_load(org)
+      }
+    else
+      require 'rake'
+      load File.join(MARQ.rootdir, 'tasks/install.rake')
+
+      @actions = [@actions] if @actions === String
+      @actions.each{|action|
+        puts "Installing #{ action }"
+        Rake::Task[action].invoke
+      }
+    end
+  end
+
+  def update
+    install
+
+    require 'rake'
+    @actions = [@actions] if @actions === String
+
+    @actions.each{|action|
+      Thread.new{
+        puts "Updating #{action}"
+        FileUtils.cd File.join(MARQ.datadir, action)
+        load File.join(MARQ.datadir, action, 'Rakefile')
+        Rake::Task['default'].invoke
+      }.join
+    }
+  end
+
+
+end
+
+SimpleConsole::Application.run(ARGV, Controller, View)
+

data/install_scripts/CustomDS/Rakefile

@@ -0,0 +1,223 @@
+require 'MARQ'
+require 'MARQ/CustomDS'
+require 'MARQ/MADB'
+require 'MARQ/annotations'
+require 'progress-monitor'
+
+$org            ||= ENV['organism']
+
+$expr_threshold ||= (ENV['threshold'] || 0.05).to_f
+$folds          ||= (ENV['folds'] || 2.5).to_f
+$nth_genes      ||= (ENV['nth_genes'] || 100).to_i
+
+$force       = [$force, ENV['force'], false].         reject{|e| e.nil?}.first 
+$update_db   = [$update_db, ENV['update_db'], true].  reject{|e| e.nil?}.first 
+$fdr         = [$fdr, ENV['fdr'], true].              reject{|e| e.nil?}.first 
+$do_folds    = [$do_folds, ENV['do_folds'], true].    reject{|e| e.nil?}.first 
+
+
+
+
+task 'data' do
+  organisms = Dir.glob(CustomDS::customdir + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - ["annotations"]
+  
+  organisms.each{|org|
+    datasets = Dir.glob(File.join(CustomDS::customdir, org) + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)}
+    if $force
+      missing = datasets
+    else
+      missing = datasets - CustomDS::datasets(org)
+    end
+    missing.each{|d|
+      CustomDS::process(d)
+      MADB::CustomDS::save(d)
+    }
+  }
+end
+
+
+def annotations(name, cross_platform = false, &block)
+  FileUtils.mkdir_p File.join("annotations", name) unless File.exist? File.join("annotations", name)
+  organisms = Dir.glob(CustomDS::customdir + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - ["annotations"]
+
+  Progress.monitor("Annotating with #{ name }")
+  organisms.each{|org|
+    Progress.monitor("Annotating organism #{ org }")
+    CustomDS::datasets(org).each{|dataset|
+      begin
+        next if File.exist? File.join("annotations", name, dataset)
+        filename = File.join("annotations", name, dataset)
+        dataset += '_cross_platform' if cross_platform && GEO.has_cross_platform?(dataset)
+        next if File.exist? CustomDS.path(dataset) + '.skip'
+        terms = block.call(dataset)
+        Open.write(filename, terms.to_yaml)
+      rescue Exception
+        puts $!.message
+      end
+    }
+  }
+end
+
+def goterms(org, list, slim, threshold)
+  return [] if list.empty?
+  results = Annotations::GO::Genecodis::Local.analysis(org, list, slim)
+  results.
+    select{|info| info[:s].to_i  > 2 }.
+    select{|info| info[:hyp_c].to_f < threshold }.
+    collect{|info| info[:items]}.collect{|id| GO::id2name(id)}
+end
+
+
+task 'annotate_Words' do
+  FileUtils.mkdir_p "annotations/Words"
+  require 'rbbt/bow/bow'
+  organisms = Dir.glob(CustomDS::customdir + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - ["annotations"]
+  organisms.each{|org|
+    CustomDS::datasets(org).each{|dataset|
+      begin
+        next if File.exist? "annotations/Words/#{ dataset }"
+        next if File.exist? CustomDS.path(dataset) + '.skip'
+
+        terms = {}
+        description = Open.read(CustomDS.path(dataset) + '.description')
+        terms[:dataset] = [dataset] +  description.words
+        Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
+          name = name.strip
+          terms[name] = name.sub(/\[ratio\]/,'').words
+        }
+        
+        Open.write("annotations/Words/#{ dataset }", terms.to_yaml)
+
+      rescue Exception
+        puts $!.message
+      end
+    }
+ }
+end
+
+task 'annotate_UMLS' do
+  FileUtils.mkdir_p "annotations/UMLS"
+  organisms = Dir.glob(CustomDS::customdir + '/*').select{|f| File.directory? f}.collect{|f| File.basename(f)} - ["annotations"]
+  organisms.each{|org|
+    CustomDS::datasets(org).each{|dataset|
+      begin
+        next if File.exist? "annotations/UMLS/#{ dataset }"
+        next if File.exist? CustomDS.path(dataset) + '.skip'
+        terms = {}
+        description = Open.read(CustomDS.path(dataset) + '.description')
+        terms[:dataset] = Annotations::UMLS::OBA(description)
+        Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
+          name = name.strip
+          terms[name] = Annotations::UMLS::OBA(name)
+        }
+        Open.write("annotations/UMLS/#{ dataset }", terms.to_yaml)
+      rescue Exception
+        puts $!.message
+      end
+    }
+  }
+
+end
+
+
+
+
+task 'annotate_Words' do
+  annotations('Words'){|dataset|
+    description = Open.read(CustomDS.path(dataset) + '.description')
+    terms[:dataset] = [dataset] +  description.words
+    Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = name.sub(/\[ratio\]/,'').words
+    }
+    terms
+  }
+end
+
+task 'annotate_UMLS' do
+  annotations('Words'){|dataset|
+    description = Open.read(CustomDS.path(dataset) + '.description')
+    terms[:dataset] = Annotations::UMLS::OBA(description).uniq
+    Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = Annotations::UMLS::OBA(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).uniq
+    }
+    terms
+  }
+end
+
+
+
+task 'annotate_Polysearch' do
+  require 'rbbt/sources/polysearch'
+  annotations('Polysearch'){|dataset|
+    terms = {}
+    description = Open.read(CustomDS.path(dataset) + '.description')
+    terms[:dataset] = Polysearch::match(description).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
+    Open.read(CustomDS.path(dataset) + '.experiments').collect{|name|
+      name = name.strip
+      terms[name] = Polysearch::match(name.sub(/.*?: /,'').sub(/\[ratio\]/,'')).values.flatten.sort.collect{|n| n.gsub(/\s+/,' ').downcase}.uniq
+    }
+    terms
+  }
+
+end
+
+task 'annotate_GO' do
+  require 'rbbt/sources/go'
+  options = { :cut_off => $expr_threshold, :fdr => $fdr, :folds => $folds, :do_folds => $do_folds, :nth_genes => $nth_genes}
+  annotations('GO_up', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    up = {}
+    genes[:up] ||= []
+    genes[:up].collect{|experiment,list|
+      up[experiment] =  goterms(org, list, false, $expr_threshold)
+    }
+    up
+  }
+
+  annotations('GO_down', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    down = {}
+    genes[:down] ||= []
+    genes[:down].collect{|experiment,list|
+      down[experiment] = goterms(org, list, false, $expr_threshold)
+    }
+    down
+  } 
+  annotations('GOSlim_up', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    up = {}
+    genes[:up] ||= []
+    genes[:up].collect{|experiment,list|
+      up[experiment] = goterms(org, list, true, $expr_threshold)
+    }
+    up
+  }
+
+  annotations('GOSlim_down', true){|dataset|
+    org = MARQ.platform_organism(dataset)
+    genes = Annotations::GO.get_genes(dataset, options)
+
+    down = {}
+    genes[:down] ||= []
+    genes[:down].collect{|experiment,list|
+      down[experiment] =  goterms(org, list, true, $expr_threshold)
+    }
+    down
+  }
+end
+
+task 'default' do
+  Rake::Task['data'].invoke
+  Rake::Task['annotate_Words'].invoke
+  Rake::Task['annotate_UMLS'].invoke
+  Rake::Task['annotate_Polysearch'].invoke
+  Rake::Task['annotate_GO'].invoke
+end