MARQ 0.0.1

data/merb/tasks/merb.thor/main.thor

@@ -0,0 +1,150 @@
+require "rubygems"
+require "rubygems/source_index"
+require "rubygems/dependency_installer"
+require "rubygems/uninstaller"
+require "fileutils"
+require File.join(File.dirname(__FILE__), "utils")
+require File.join(File.dirname(__FILE__), "gem_ext")
+require File.join(File.dirname(__FILE__), "ops")
+
+$INSTALLING = []
+
+module Merb
+  
+  class Gem < Thor
+    extend ColorfulMessages
+    
+    def initialize
+      dirs = [Dir.pwd, File.dirname(__FILE__) / ".."]
+      root = dirs.find {|d| File.file?(d / "config" / "dependencies.rb")}
+      
+      if root
+        @depsrb = root / "config" / "dependencies.rb"
+      else
+        self.class.error "dependencies.rb was not found"
+        exit!
+      end
+      
+      FileUtils.mkdir_p(Dir.pwd / "gems")
+      
+      @list = Collector.collect(File.read(@depsrb))
+      @idx = ::Gem::SourceIndex.new.load_gems_in("gems/specifications")
+    end
+    
+    def list
+      require "pp"
+      pp @list
+    end
+    
+    desc "redeploy", "Syncs up gems/cache with gems/gems. All gems in the cache " \
+                     "that are not already installed will be installed from the " \
+                     "cache. All installed gems that are not in the cache will " \
+                     "be uninstalled."
+    def redeploy
+      gem_dir = Dir.pwd / "gems" / "gems"
+      cache_dir = Dir.pwd / "gems" / "cache"
+      
+      gems  = Dir[gem_dir / "*"].map!  {|n| File.basename(n)}
+      cache = Dir[cache_dir / "*.gem"].map! {|n| File.basename(n, ".gem")}
+      new_gems = cache - gems
+      outdated = gems - cache
+      idx = ::Gem::SourceIndex.new
+      idx.load_gems_in(Dir.pwd / "gems" / "specifications")
+
+      new_gems.each do |g|
+        installer = ::Gem::Installer.new(cache_dir / "#{g}.gem",
+          :bin_dir => Dir.pwd / "bin",
+          :install_dir => Dir.pwd / "gems",
+          :ignore_dependencies => true,
+          :user_install => false,
+          :wrappers => true,
+          :source_index => idx)
+            
+        installer.install
+      end
+      
+      outdated.each do |g|
+        /(.*)\-(.*)/ =~ g
+        name, version = $1, $2
+        uninstaller = ::Gem::Uninstaller.new(name,
+          :version => version,
+          :bin_dir => Dir.pwd / "bin",
+          :install_dir => Dir.pwd / "gems",
+          :ignore => true,
+          :executables => true
+        )
+        uninstaller.uninstall
+      end
+    end
+    
+    desc "confirm", "Confirm the current setup. merb:gem:install will " \
+                    "automatically run this task before committing the " \
+                    "changes it makes."
+    def confirm(gems = @list)
+      ::Gem.path.replace([Dir.pwd / "gems"])
+      ::Gem.source_index.load_gems_in(Dir.pwd / "gems" / "specifications")
+      
+      self.class.info "Confirming configuration..."
+      
+      ::Gem.loaded_specs.clear
+      
+      begin
+        gems.each do |name, versions|
+          versions ||= []
+          ::Gem.activate name, *versions
+        end
+      rescue ::Gem::LoadError => e
+        self.class.error "Configuration could not be confirmed: #{e.message}"
+        self.class.rollback_trans
+      end
+      self.class.info "Confirmed"
+    end
+    
+    desc 'install', 'Sync up your bundled gems with the list in config/dependencies.rb'
+    def install(*gems)
+      if gems.empty?
+        gems = @list
+      else
+        gems = gems.map {|desc| name, *versions = desc.split(" ") }
+      end
+      
+      $GEMS = gems
+      
+      self.class.begin_trans
+      
+      gems.each do |name, versions|
+        dep = ::Gem::Dependency.new(name, versions || [])
+        unless @idx.search(dep).empty?
+          next
+        end
+        
+        rescue_failures do
+          $INSTALLING = dep
+          _install(dep)
+        end
+      end
+
+      gem_dir = Dir.pwd / "gems" / "gems"
+      installed_gems  = Dir[gem_dir / "*"].map!  {|n| File.basename(n)}
+      
+      list = full_list.map {|x| x.full_name}.compact
+      
+      (installed_gems - list).each do |g|
+        /^(.*)\-(.*)$/ =~ g
+        name, version = $1, $2
+        uninstaller = ::Gem::Uninstaller.new(name,
+          :version => version,
+          :bin_dir => (Dir.pwd / "bin").to_s,
+          :install_dir => (Dir.pwd / "gems").to_s,
+          :ignore => true,
+          :executables => true
+        )
+        uninstaller.uninstall
+      end
+      
+      confirm(gems)
+      
+      self.class.commit_trans
+    end
+  end
+end

data/merb/tasks/merb.thor/ops.rb

@@ -0,0 +1,93 @@
+module Thor::Tasks
+  module Merb
+    class Collector
+      attr_reader :dependencies
+
+      def self.collect(str)
+        collector = new
+        collector.instance_eval(str)
+        collector.dependencies
+      end
+
+      def initialize
+        @dependencies = []
+      end
+
+      def dependency(name, *versions)
+        versions.pop if versions.last.is_a?(Hash)
+        @dependencies << [name, versions]
+      end
+    end
+    
+    class Gem < Thor
+      def full_list
+        @idx.load_gems_in("gems/specifications")
+
+        @list.map do |name, versions|
+          dep = ::Gem::Dependency.new(name, versions)
+          spec = @idx.search(dep).last
+          unless spec
+            self.class.error "A required dependency #{dep} was not found"
+            self.class.rollback_trans
+          end
+          deps = spec.recursive_dependencies(dep, @idx)
+          [spec] + deps
+        end.flatten.uniq
+      end
+      
+      def rescue_failures(error = StandardError, prc = nil)
+        begin
+          yield
+        rescue error => e
+          if prc
+            prc.call(e)
+          else
+            puts e.message
+            puts e.backtrace
+          end
+          self.class.rollback_trans
+        end
+      end
+
+      def self.begin_trans
+        note "Beginning transaction"
+        FileUtils.cp_r(Dir.pwd / "gems", Dir.pwd / ".original_gems")
+      end
+
+      def self.commit_trans
+        note "Committing transaction"
+        FileUtils.rm_rf(Dir.pwd / ".original_gems")
+      end
+
+      def self.rollback_trans
+        if File.exist?(Dir.pwd / ".original_gems")
+          note "Rolling back transaction"
+          FileUtils.rm_rf(Dir.pwd / "gems")
+          FileUtils.mv(Dir.pwd / ".original_gems", Dir.pwd / "gems")
+        end
+        exit!
+      end
+      
+      private
+      def _install(dep)
+        @idx.load_gems_in("gems/specifications")
+        return if @idx.search(dep).last
+        
+        installer = ::Gem::DependencyInstaller.new(
+          :bin_dir => Dir.pwd / "bin",
+          :install_dir => Dir.pwd / "gems",
+          :user_install => false)
+
+        begin
+          installer.install dep.name, dep.version_requirements
+        rescue ::Gem::GemNotFoundException => e
+          puts "Cannot find #{dep}"
+        rescue ::Gem::RemoteFetcher::FetchError => e
+          puts e.message
+          puts "Retrying..."
+          retry
+        end
+      end
+    end
+  end
+end

data/merb/tasks/merb.thor/utils.rb

@@ -0,0 +1,40 @@
+class String
+  def /(other)
+    (Pathname.new(self) + other).to_s
+  end
+end
+
+module ColorfulMessages
+  
+  # red
+  def error(*messages)
+    puts messages.map { |msg| "\033[1;31m#{msg}\033[0m" }
+  end
+  
+  # yellow
+  def warning(*messages)
+    puts messages.map { |msg| "\033[1;33m#{msg}\033[0m" }
+  end
+  
+  # green
+  def success(*messages)
+    puts messages.map { |msg| "\033[1;32m#{msg}\033[0m" }
+  end
+  
+  alias_method :message, :success
+  
+  # magenta
+  def note(*messages)
+    puts messages.map { |msg| "\033[1;35m#{msg}\033[0m" }
+  end
+  
+  # blue
+  def info(*messages)
+    puts messages.map { |msg| "\033[1;34m#{msg}\033[0m" }
+  end
+  
+end
+
+module ThorUI
+  extend ColorfulMessages
+end

data/tasks/install.rake

@@ -0,0 +1,21 @@
+require 'MARQ'
+
+$datadir = MARQ.datadir
+$scriptdir = File.join(MARQ.rootdir, '/install_scripts')
+
+task 'GEO' do |t|
+    directory = "#{$datadir}/GEO"
+    FileUtils.mkdir_p(directory)
+    %w(Rakefile series).each{|f|
+      FileUtils.cp_r File.join($scriptdir, "GEO/#{ f }"), directory 
+    }
+    FileUtils.mkdir(File.join(directory, "platforms")) unless File.exist? File.join(directory, "platforms")
+end
+
+task 'CustomDS' do |t|
+    directory = "#{$datadir}/CustomDS"
+    FileUtils.mkdir_p(directory)
+    Dir.glob($scriptdir + '/CustomDS/*').each{|f|
+      FileUtils.cp_r  f, directory 
+    }
+end

data/webservice/MARQWS.rb

@@ -0,0 +1,167 @@
+#!/usr/bin/ruby 
+
+require 'MARQ'
+require 'MARQ/main'
+require 'MARQ/annotations'
+require 'simplews/jobs'
+require 'zlib'
+require 'zaml'
+
+class MARQWS < SimpleWS::Jobs
+  class ArgumentError < Exception; end
+  class NotDone < Exception; end 
+
+  helper :gzip do |string|
+    ostream = StringIO.new
+    begin
+      gz = Zlib::GzipWriter.new(ostream)
+      gz.write(string)
+      ostream.string
+    ensure
+      gz.close
+    end
+  end
+
+  helper :gunzip do |file|
+    begin
+      gz = Zlib::GzipReader.new(File.open(file))
+      gz.read
+    end
+  end
+
+  def logratios(dataset, comparison, genes)
+    path = MARQ.dataset_path(dataset)
+
+    position = File.open(path + '.experiments').collect{|l| l.chomp.strip}.index(comparison.chomp.strip)
+    raise ArgumentError, "Comparison #{ comparison.chomp.strip } not found" if position.nil?
+
+    `paste #{path + '.codes'} #{path + '.logratios'}| grep '#{genes.collect{|gene| "^#{ gene }[[:space:]]"}.join('\|')}'|cut -f1,#{position + 2}`
+  end
+
+  def ts(dataset, comparison, genes)
+    path = MARQ.dataset_path(dataset)
+    if comparison.match(/\[ratio\]/) 
+      raise ArgumentError, "Comparison  #{ comparison.chomp.strip } does not have a t value"
+    end
+
+    position = File.open(path + '.experiments').collect{|l| l.chomp.strip}.select{|name| !name.match(/\[ratio\]/)}.index(comparison.chomp.strip)
+    raise ArgumentError, "Comparison #{ comparison.chomp.strip } not found" if position.nil?
+
+    `paste #{path + '.codes'} #{path + '.t'}| grep '#{genes.collect{|gene| "^#{ gene }[[:space:]]"}.join('\|')}'|cut -f1,#{position + 2}`
+  end
+
+ 
+  task :annotations, %w(job type), {:job => :string, :type => :string}, ["annotations/{JOB}.yaml", "annotations/{JOB}_enriched.yaml"] do  |job, type|
+    step(:scores, "Loading Scores")
+    scores = YAML::load(gunzip(File.join(workdir, "scores/#{job}.yaml.gz")))
+    
+    step(:annotations, "Getting annotations")
+    annotations, terms = Annotations.annotations(scores, type, 0.05, :rank)
+
+    step(:saving, "Saving")
+    
+    write("annotations/#{job_name}.yaml", gzip(ZAML.dump(annotations)))
+    write("annotations/#{job_name}_enriched.yaml", gzip(ZAML.dump(terms)))
+  end
+
+  task :match_platform, %w(platform up down), {:platform => :string, :up => :array, :down => :array},
+    ['scores/{JOB}.yaml.gz' ] do |platform, up, down|
+
+    up = up.collect{|gene| gene.strip if gene}
+    down = down.collect{|gene| gene.strip if gene}
+
+    info(:platform => platform,
+         :organism => MARQ.platform_organism(platform),
+         :up => up,
+         :down => down
+        )
+
+    raise ArgumentError, "No genes identified" if up.empty? && down.empty?
+    step(:matching, "Matching genes to platform #{ platform }, up: #{up.length}, down:  #{down.length}")
+    scores = MARQ.platform_scores_up_down(platform, up, down)
+
+    raise ArgumentError, "No genes matched in platform #{ platform }, check that they are in the right format, automatic transalation is not performed in platform specific queries" if scores.keys.empty?
+
+    step(:saving, "Saving results")
+    write("scores/#{job_name}.yaml.gz", gzip(ZAML.dump(scores)))
+  end
+
+  task :match_organism, %w(organism up down), {:organism => :string, :up => :array, :down => :array},
+    ['scores/{JOB}.yaml.gz'] do |organism, up, down|
+    require 'MARQ/ID'
+
+    up = up.collect{|gene| gene.strip if gene}
+    down = down.collect{|gene| gene.strip if gene}
+
+    step(:translating, "Translating genes")
+
+    cross_platform_up = ID.translate(organism, up)
+    cross_platform_down = ID.translate(organism, down)
+
+    info(:organism => organism,
+         :up => up,
+         :down => down,
+         :cross_platform_up => cross_platform_up.collect{|n| n || "NO MATCH"},
+         :cross_platform_down => cross_platform_down.collect{|n| n || "NO MATCH"})
+
+    cross_platform_up = cross_platform_up.compact
+    cross_platform_down = cross_platform_down.compact
+    
+    raise ArgumentError, "No genes identified" if cross_platform_up.empty? && cross_platform_down.empty?
+
+    step(:matching, "Matching genes to organism #{ organism }, up: #{cross_platform_up.length}, down:  #{cross_platform_down.length}")
+    
+    platforms = MARQ::organism_platforms(organism).select{|platform| MARQ.has_cross_platform? nil, platform }.collect{|platform| platform + "_cross_platform"}
+
+    scores = {}
+    step(:processing, "Procesing #{platforms.length} datasets")
+    platforms.each_with_index{|platform, i |
+      step(:processing, "Generating Scores for platform #{ platform } #{ i + 1 }/#{platforms.length}.")
+      scores = scores.merge(MARQ.platform_scores_up_down(platform, cross_platform_up, cross_platform_down))
+    }
+    step(:saving, "Saving results")
+    write("scores/#{job_name}.yaml.gz", gzip(ZAML.dump(scores)))
+  end
+
+  
+  
+  class Scheduler::Job
+    alias_method :old_step, :step
+    def step(status, message=nil)
+      puts "#{Time.now} => [#{ @name }]: #{ status }. #{ message }"
+      old_step(status, message)
+    end 
+  end
+
+  def initialize(*args)
+    super(*args)
+    @soaplet.allow_content_encoding_gzip = true
+
+    serve :logratios, %w(dataset comparison genes), {:dataset => :string, :comparison => :string, :genes => :array, :return => :string}
+    serve :ts, %w(dataset comparison genes), {:dataset => :string, :comparison => :string, :genes => :array, :return => :string}
+    FileUtils.mkdir(File.join(workdir, 'scores')) unless File.exist?(File.join(workdir, 'scores'))
+    FileUtils.mkdir(File.join(workdir, 'annotations')) unless File.exist?(File.join(workdir, 'annotations'))
+  end
+
+
+end
+
+if __FILE__ == $0
+
+  puts "Starting WS"
+  host = ARGV[0] || `hostname`.chomp.strip + '.' +  `hostname -d`.chomp.strip
+  port = ARGV[1] || '8282'
+
+  puts "Starting Server in #{ host }:#{ port }"
+  server = MARQWS.new("MARQ", "MARQ Web Server",host, port, File.join(MARQ.workdir,'webservice', 'jobs'))
+
+  FileUtils.mkdir_p File.join(MARQ.workdir, '/webservice', 'wsdl/') unless File.exist? File.join(MARQ.workdir, '/webservice', 'wsdl/')
+  File.open(File.join(MARQ.workdir, '/webservice', 'wsdl', 'MARQWS.wsdl'),'w'){|file| file.write server.wsdl}
+
+  trap('INT') { server.shutdown }
+  server.start
+
+end
+
+
+