quorum 0.1.0 → 0.2.0

data/spec/dummy/quorum/bin/fetch

@@ -0,0 +1,115 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+
+require 'trollop'
+require 'fetch_tools/blast_db'
+
+FETCH_TOOLS = {
+  "blastdbcmd" => ["blastdbcmd"]
+}
+
+module Quorum
+  #
+  # Parse Options for Quorum Fetch
+  #
+  class Fetch
+
+    def initialize
+      opts = Trollop::options do
+        text <<-HEAD
+
+Quorum
+
+A flexible bioinformatics search tool.
+
+Options:
+
+        HEAD
+
+        # Fetch tool
+        opt :fetch_tool, "Fetch tool to execute. Available tools: " <<
+        "#{FETCH_TOOLS.keys.join(', ')}", :type => :string,
+          :required => true, :short => "-f"
+
+        # General settings
+        opt :log_directory, "Path to log directory", :type => :string,
+          :required => true, :short => "-l"
+        opt :tmp_directory, "Path to tmp directory", :type => :string,
+          :required => true, :short => "-m"
+        
+        # Global settings
+        opt :fetch_database, "Path to fetch tool database", :type => :string,
+          :short => "-d"
+        opt :fetch_database_names, "Semicolon delimited database names",
+          :type => :string, :short => "-n"
+
+        # Algorithm specific opts
+        opt :blast_hit_id, "Blast hit id", :type => :string, :short => "-b"
+        opt :blast_hit_display_id, "Blast hit display id", :type => :string, 
+          :short => "-s"
+        opt :blast_algo, "Blast algorithm", :type => :string, :short => "-a"
+      end
+
+      @args = {} # Contains valid opts.
+
+      ## Additional opt validation. ##
+
+      unless FETCH_TOOLS.keys.include?(opts[:fetch_tool].downcase)
+        Trollop::die :fetch_tool, "fetch_tool not found\n" <<
+        "Available tools: " + FETCH_TOOLS.keys.join(", ")
+      end
+
+      @args[:fetch_tool] = opts[:fetch_tool].downcase
+
+      unless opts[:fetch_database] && Dir.exists?(opts[:fetch_database])
+        Trollop::die :fetch_database, "fetch tool database not found\n" <<
+        "Supplied directory: " + opts[:fetch_database]
+      end
+
+      @args[:fetch_database]       = opts[:fetch_database]
+      @args[:fetch_database_names] = opts[:fetch_database_names]
+
+      unless Dir.exists?(opts[:log_directory])
+        Trollop::die :log_directory, "log directory not found\n" <<
+        "Supplied directory: " + opts[:log_directory]
+      end
+
+      @args[:log_directory]         = opts[:log_directory]
+      @args[:tmp_directory]         = opts[:tmp_directory]
+      @args[:blast_hit_id]          = opts[:blast_hit_id]
+      @args[:blast_hit_display_id]  = opts[:blast_hit_display_id]
+      @args[:blast_algo]            = opts[:blast_algo]
+
+      ## Check System Dependancies ##
+
+      check_dependencies
+
+      ## Execute fetch tool ##
+
+      if @args[:fetch_tool].include? "blastdbcmd"
+        blast = Quorum::FetchTools::BlastDb.new(@args)
+        blast.execute_blast_db_cmd
+      end
+    end
+
+    #
+    # Check system dependencies.
+    #
+    def check_dependencies
+      FETCH_TOOLS[@args[:fetch_tool]].each do |f|
+        system("which #{f} > /dev/null 2>&1")
+        if $?.exitstatus > 0
+          Trollop::die "Quorum dependency not found for tool " <<
+          "#{@args[:fetch_tool].to_s}. Please add `#{f}` to your PATH."
+        end
+      end
+    end
+
+  end
+end
+
+if __FILE__ == $0
+  Quorum::Fetch.new  
+end
+

data/spec/dummy/quorum/lib/fetch_tools/blast_db.rb

@@ -0,0 +1,106 @@
+$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
+
+require 'logger'
+
+module Quorum
+  module FetchTools
+    #
+    # Blast DB Fetch Tool
+    #
+    class BlastDb
+
+      # blastdbcmd -dbtype
+      DB_TYPE = {
+        "blastn"  => "nucl",
+        "blastx"  => "prot",
+        "tblastn" => "nucl",
+        "blastp"  => "prot"
+      }
+
+      private
+
+      def initialize(args)
+        @tool           = args[:fetch_tool]
+        @entry          = args[:blast_hit_id]
+        @hit_display_id = args[:blast_hit_display_id]
+        @blast_algo     = args[:blast_algo]
+        @log_directory  = args[:log_directory]
+        @tmp_directory  = args[:tmp_directory]
+        @fetch_database = args[:fetch_database]
+        @databases      = args[:fetch_database_names]
+
+        @logger = Quorum::Logger.new(@log_directory)
+
+        @db = @databases.split(';')
+        @db.map! { |d| File.join(@fetch_database, d) }
+
+        @db_type = DB_TYPE[@blast_algo]
+        @db_type ||= 'guess'
+      end
+
+      #
+      # Generate the blastdbcmd(s).
+      #
+      def generate_blast_db_cmds
+        @cmds = []
+        @db.each do |d|
+          blastdbcmd = "blastdbcmd " <<
+          "-db '#{d}' " <<
+          "-dbtype '#{@db_type}' " <<
+          "-entry '#{@entry}' "
+          @cmds << blastdbcmd
+        end
+      end
+
+      #
+      # Parse the blastdbcmd returned sequence(s) and delete if @hit_display_id
+      # is not present.
+      #
+      def parse_and_send_results
+        seqs = @seqs.split('>')
+        seqs.delete_if { |s| s.empty? || !s.include?(@hit_display_id) }
+        if seqs.length != 1
+          @logger.log(
+            "Quorum::FetchTools::BlastDb#parse_and_send_results",
+            "blastdbcmd returned #{seqs.length} sequence(s). Please ensure " <<
+            "your Blast database source FASTA headers are unique across ALL " <<
+            "databases. See the entry above for details."
+          )
+          "An error occurred while processing your request."
+        else
+          ">" << seqs.first
+        end
+      end
+
+      public
+
+      #
+      # Execute the blastdbcmd(s) and return the matching sequence.
+      #
+      # To make Blast execute as quickly as possible, each selected 
+      # algorithm blasts against all supplied databases at once. 
+      # 
+      # See quorum/lib/search_tools/blast.rb for more information. 
+      #
+      # One consequence of this is not knowing which Blast database to use
+      # when retrieving a hit sequence via blastdbcmd.
+      #
+      # See blastdbcmd -help for more information.
+      #
+      # To work around this issue, simply execute blastdbcmd against all
+      # Blast databases and filter on hit_display_id.
+      #
+      def execute_blast_db_cmd
+        generate_blast_db_cmds
+        @logger.log("NCBI Blast", @cmds.join('; '))
+        
+        @seqs = ""
+        @cmds.each { |c| @seqs << `#{c} 2> /dev/null` }
+        
+        seq = parse_and_send_results
+        $stdout.print seq
+      end
+
+    end
+  end
+end

data/spec/dummy/quorum/lib/search_tools/blast.rb

@@ -11,6 +11,8 @@ module Quorum
     class Blast
 
       class QuorumJob < ActiveRecord::Base
+        set_table_name "quorum_jobs"
+
         has_one :quorum_blastn_job, 
           :foreign_key => "job_id"
         has_many :quorum_blastn_job_reports, 
@@ -33,38 +35,46 @@ module Quorum
       end
 
       class QuorumBlastnJob < ActiveRecord::Base
+        set_table_name "quorum_blastn_jobs"
         belongs_to :quorum_job
         has_many :quorum_blastn_job_reports
       end
 
       class QuorumBlastnJobReport < ActiveRecord::Base
+        set_table_name "quorum_blastn_job_reports"
         belongs_to :quorum_blastn_job
       end
 
       class QuorumBlastxJob < ActiveRecord::Base
+        set_table_name "quorum_blastx_jobs"
         belongs_to :quorum_job
         has_many :quorum_blastx_job_reports
       end
 
       class QuorumBlastxJobReport < ActiveRecord::Base
+        set_table_name "quorum_blastx_job_reports"
         belongs_to :quorum_blastx_job
       end
 
       class QuorumTblastnJob < ActiveRecord::Base
+        set_table_name "quorum_tblastn_jobs"
         belongs_to :quorum_job
         has_many :quorum_tblastn_job_reports
       end
 
       class QuorumTblastnJobReport < ActiveRecord::Base
+        set_table_name "quorum_tblastn_job_reports"
         belongs_to :quorum_tblastn_job
       end
 
       class QuorumBlastpJob < ActiveRecord::Base
+        set_table_name "quorum_blastp_jobs"
         belongs_to :quorum_job
         has_many :quorum_blastp_job_reports
       end
 
       class QuorumBlastpJobReport < ActiveRecord::Base
+        set_table_name "quorum_blastp_job_reports"
         belongs_to :quorum_blastp_job
       end
 
@@ -235,26 +245,26 @@ module Quorum
       end
 
       #
-      # Format Blast report hit_id and hit_def.
+      # Format Blast report hit_display_id and hit_def.
       #
       # For added flexibility, Quorum doesn't parse seqids or deflines.
       # Instead, format_hit splits the hit_def on whitespace and
-      # reports hit_id as the first element and hit_def as the second.
+      # reports hit_display_id as the first element and hit_def as the second.
       #
       def format_hit(str)
-        hit_id  = ""
+        hit_display_id  = ""
         hit_def = ""
 
         hit = str.split(" ", 2)
         if hit.length < 2
-          hit_id  = hit.first
+          hit_display_id  = hit.first
           hit_def = "None"
         else
-          hit_id  = hit.first
+          hit_display_id  = hit.first
           hit_def = hit.last
         end
 
-        return hit_id, hit_def
+        return hit_display_id, hit_def
       end
 
       #
@@ -275,8 +285,9 @@ module Quorum
             @data[:query_len] = iteration.query_len
 
             iteration.each do |hit|
-              @data[:hit_id], @data[:hit_def] = format_hit(hit.hit_def)
+              @data[:hit_display_id], @data[:hit_def] = format_hit(hit.hit_def)
 
+              @data[:hit_id]        = hit.hit_id
               @data[:hit_accession] = hit.accession
               @data[:hit_len]       = hit.len
 

data/spec/models/job_spec.rb

@@ -4,6 +4,7 @@ describe Quorum::Job do
   
   before(:each) do
     @job = Quorum::Job.new()
+    ResqueSpec.reset!
   end
 
   it "fails validation without params (using error_on)" do
@@ -22,9 +23,6 @@ describe Quorum::Job do
   end
 
   it "queues workers after save" do
-    resque = double("Resque")
-    resque.stub(:enqueue)
-  
     @job.sequence = File.open(
       File.expand_path("../../data/nucl_prot_seqs.txt", __FILE__)
     ).read
@@ -33,8 +31,9 @@ describe Quorum::Job do
     @job.blastn_job.queue     = true
     @job.blastn_job.blast_dbs = ["tmp"]
 
-    @job.should_receive(:queue_workers)
     @job.save!
+
+    Workers::System.should have_queue_size_of(1)
   end
 
 end

data/spec/quorum_installed_spec.rb

@@ -61,4 +61,9 @@ describe "Quorum" do
     f = File.open(File.join(@dummy_path, "config", "routes.rb"), "r")
     f.read.include?("mount Quorum::Engine => \"/quorum\"").should be_true
   end
+
+  it "ensures Resque::Server is mounted in dummy/config/routes.rb" do
+    f = File.open(File.join(@dummy_path, "config", "routes.rb"), "r")
+    f.read.include?("mount Resque::Server.new, :at => \"/quorum/resque\"").should be_true
+  end
 end

data/spec/requests/jobs_spec.rb

@@ -51,8 +51,12 @@ describe "Jobs" do
     before(:all) do
       Capybara.current_driver = :selenium
     end
+    before(:each) do
+      ResqueSpec.reset!
+      ResqueSpec.inline = true
+    end
     describe "submit sequences in attached file" do
-	    it "check Blast algorithms, fill in default values and view query results" do
+	    it "check algorithms, fill in values, view results and download hit sequence" do
 	      visit new_job_path
 	      current_path.should eq(new_job_path)
 	
@@ -109,13 +113,22 @@ describe "Jobs" do
         click_link "Tblastn"
 	      page.should have_content("Your search returned 0 hits.") 
 
+        ## Interact with the Blast results. ##
         click_link "Blastn"
+
+        # Render modal box.
 	      find("#blastn-results").find("td a").click
         page.should have_content("Quorum Report Details")
         page.should have_content("qseq")
         page.should have_content("hseq")
+        # Download sequence
+        find("p.small a#download_sequence_1").click
+        page.should have_content("Fetching sequence...")
+        page.should have_content("Sequence Downloaded Successfully")
 
         click_link "Blastp"
+
+        # Render modal box.
 	      find("#blastp-results").find("td a").click
         page.should have_content("Quorum Report Details")
         page.should have_content("qseq")

data/spec/spec_helper.rb

@@ -5,6 +5,7 @@ require 'rspec/rails'
 require 'capybara/rspec'
 require 'factory_girl'
 require 'database_cleaner'
+require 'resque_spec'
 
 FactoryGirl.definition_file_paths << File.join(File.dirname(__FILE__), 'factories')
 FactoryGirl.find_definitions
@@ -13,7 +14,7 @@ ENGINE_RAILS_ROOT = File.expand_path("../../", __FILE__)
 
 # Requires supporting ruby files with custom matchers and macros, etc,
 # in spec/support/ and its subdirectories.
-Dir[Rails.root.join("spec/support/**/*.rb")].each {|f| require f}
+Dir[Rails.root.join("spec/support/**/*.rb")].each { |f| require f }
 
 RSpec.configure do |config|
   config.mock_with :rspec
@@ -42,4 +43,34 @@ RSpec.configure do |config|
   config.after(:each) do
     DatabaseCleaner.clean
   end
+
+  ## Redis ##
+  REDIS_PID        = File.expand_path("../dummy/tmp/pids/redis-test.pid", __FILE__)
+  REDIS_CACHE_PATH = File.expand_path("../dummy/tmp/cache/", __FILE__)
+
+  config.before(:suite) do
+    redis_options = {
+      "daemonize"     => "yes",
+      "pidfile"       => REDIS_PID,
+      "port"          => 9736,
+      "timeout"       => 300,
+      "save 900"      => 1,
+      "save 300"      => 1,
+      "save 60"       => 10000,
+      "dbfilename"    => "dump.rdb",
+      "dir"           => REDIS_CACHE_PATH,
+      "loglevel"      => "debug",
+      "logfile"       => "stdout",
+      "databases"     => 16
+    }.map { |k, v| "#{k} #{v}" }.join('
+                                      ')
+    `echo '#{redis_options}' | redis-server -`
+  end
+
+  config.after(:suite) do
+    %x{
+      cat #{REDIS_PID} | xargs kill -QUIT
+      rm -f #{REDIS_CACHE_PATH}dump.rdb
+    }
+  end 
 end

data/spec/support/streams.rb

@@ -0,0 +1,16 @@
+require 'stringio'
+
+#
+# Capture STDOUT and STDERR
+#
+def capture(*streams)
+  streams.map! { |stream| stream.to_s }
+  begin
+    result = StringIO.new
+    streams.each { |stream| eval "$#{stream} = result" }
+    yield
+  ensure
+    streams.each { |stream| eval("$#{stream} = #{stream.upcase}") }
+  end
+  result.string
+end

data/spec/templates/blast_db_spec.rb

@@ -0,0 +1,63 @@
+require 'spec_helper'
+require File.expand_path("../../dummy/quorum/lib/fetch_tools/blast_db", __FILE__)
+
+describe "Quorum::FetchTools::BlastDb" do
+  describe "#execute_blast_db_cmd" do
+    before(:each) do
+      # Set the args as though we executed bin/fetch
+      @args = {
+        :fetch_tool           => "blastdbcmd",
+        :blast_hit_id         => nil,
+        :blast_hit_display_id => nil,
+        :blast_algo           => nil,
+        :fetch_database_names => "test",
+
+        :log_directory   => File.expand_path(
+          "../../dummy/quorum/log", __FILE__
+        ),
+        :tmp_directory   => File.expand_path(
+          "../../dummy/quorum/tmp", __FILE__
+        ),
+        :fetch_database  => File.expand_path(
+          "../../dummy/quorum/blastdb", __FILE__
+        )
+      }
+    end
+
+    it "executes blastdbcmd for blastn and returns correct sequence" do
+      @args[:blast_hit_id]         = "gnl|BL_ORD_ID|0"
+      @args[:blast_hit_display_id] = "TOG900080"
+      @args[:blast_algo]           = "blastn"
+
+      seqs = File.readlines(
+        File.expand_path("../../data/nucl_seqs.txt", __FILE__)
+      )
+      seqs[0].gsub!(">", "")
+      seqs.insert(0, ">" + @args[:blast_hit_id] + " ")
+
+      # Sequence #execute_blast_db_cmd should return.
+      seq = seqs.slice(0,3).join("").gsub("\n", "")
+
+      fetch = Quorum::FetchTools::BlastDb.new(@args)
+      lambda {
+        output = capture(:stdout) {
+          fetch.execute_blast_db_cmd
+        }
+        output.gsub("\n", "").should eq(seq)
+      }.should_not raise_error
+    end
+
+    it "returns error message if filtered sequence.length != 1" do
+      @args[:blast_hit_id]         = "null"
+      @args[:blast_hit_display_id] = "null"
+      @args[:blast_algo]           = "blastp"
+      fetch = Quorum::FetchTools::BlastDb.new(@args)
+      lambda {
+        output = capture(:stdout) {
+          fetch.execute_blast_db_cmd
+        }
+        output.should eq("An error occurred while processing your request.")
+      }.should_not raise_error
+    end
+  end
+end

data/spec/templates/blast_spec.rb

@@ -1,8 +1,114 @@
 require 'spec_helper'
-require 'generators/templates/blast'
+require File.expand_path("../../dummy/quorum/lib/search_tools/blast", __FILE__)
 
-# Let the request specs handle Quorum::SearchTools
 describe "Quorum::SearchTools::Blast" do
   describe "#execute_blast" do
+    before(:each) do
+      # Set args as though we executed bin/search.
+      @args = {
+        :search_tool     => "blastn",
+        :id              => nil,
+        :log_directory   => File.expand_path(
+          "../../dummy/quorum/log", __FILE__
+        ),
+        :tmp_directory   => File.expand_path(
+          "../../dummy/quorum/tmp", __FILE__
+        ),
+        :search_database => File.expand_path(
+          "../../dummy/quorum/blastdb", __FILE__
+        ),
+        :threads         => 1
+      }
+
+      @job = Quorum::Job.new()
+
+      @job.sequence = File.open(
+        File.expand_path("../../data/nucl_prot_seqs.txt", __FILE__)
+      ).read
+
+      @job.build_blastn_job
+      @job.blastn_job.queue     = true
+      @job.blastn_job.blast_dbs = ["test"]
+
+      @job.build_blastx_job
+      @job.blastx_job.queue     = true
+      @job.blastx_job.blast_dbs = ["test"]
+
+      @job.build_tblastn_job
+      @job.tblastn_job.queue     = true
+      @job.tblastn_job.blast_dbs = ["test"]
+
+      @job.build_blastp_job
+      @job.blastp_job.queue     = true
+      @job.blastp_job.blast_dbs = ["test"]
+    end
+  
+    it "executes blastn on a given dataset" do
+      @job.stub(:queue_workers)
+      @job.save!
+
+      @args[:id] = @job.id
+
+      blast = Quorum::SearchTools::Blast.new(@args)
+      expect {
+        blast.execute_blast
+      }.to_not raise_error
+
+      Dir.glob(
+        File.join(@args[:tmp_directory], "*")
+      ).length.should be == 0
+    end 
+
+    it "executes blastx on a given dataset" do
+      @job.stub(:queue_workers)
+      @job.save!
+
+      @args[:search_tool] = "blastx"
+      @args[:id]          = @job.id
+
+      blast = Quorum::SearchTools::Blast.new(@args)
+      expect {
+        blast.execute_blast
+      }.to raise_error(SystemExit)
+
+      Dir.glob(
+        File.join(@args[:tmp_directory], "*")
+      ).length.should be == 0
+    end 
+
+    it "executes tblastn on a given dataset" do
+      @job.stub(:queue_workers)
+      @job.save!
+
+      @args[:search_tool] = "tblastn"
+      @args[:id]          = @job.id
+
+      blast = Quorum::SearchTools::Blast.new(@args)
+      expect {
+        blast.execute_blast
+      }.to raise_error(SystemExit)
+
+      Dir.glob(
+        File.join(@args[:tmp_directory], "*")
+      ).length.should be == 0
+    end 
+
+    it "executes blastp on a given dataset" do
+      @job.stub(:queue_workers)
+      @job.save!
+
+      @args[:search_tool] = "blastp"
+      @args[:id]          = @job.id
+
+      blast = Quorum::SearchTools::Blast.new(@args)
+      expect {
+        blast.execute_blast
+      }.to_not raise_error
+
+      Dir.glob(
+        File.join(@args[:tmp_directory], "*")
+      ).length.should be == 0
+    end 
+
   end
 end