sequenceserver-beta 0.8.7.beta1

data/lib/sequenceserver/blast.rb

@@ -0,0 +1,92 @@
+require 'tempfile'
+
+module SequenceServer
+  # Simple BLAST+ wrapper.
+  class Blast
+
+    ERROR_LINE = /\(CArgException.*\)\s(.*)/
+
+    # command string to be executed
+    attr_reader :command
+
+    # result of executing command
+    attr_reader :result
+
+    # errors as [status, message]
+    attr_reader :error
+
+    # Initialize a new blast search.
+    # ---
+    # Arguments:
+    # * method (String)    - blast executable (shell executable, or absolute path)
+    # * query (String)     - query string
+    # * databases (String) - database name as returned by 'blastdbcmd -list'
+    # * options (String)   - other options
+    #
+    # ---
+    # Examples:
+    #
+    #   b = Blast.new("blastn", 'ATGTCCGCGAATCGATTGAACGTGCTGGTGACCCTGATGCTCGCCGTCGCGCTTCTTGTG', "S.cdna.fasta",  "-html -num_threads 4")
+    #
+    #   b.run!        => true
+    #   b.result      => "blast output"
+    def initialize(method, query, databases, options = nil)
+      @method    = method
+      @databases = databases
+
+      # create a tempfile for the given query
+      @qfile = Tempfile.new('sequenceserver_query')
+      @qfile.puts(query)
+      @qfile.close
+
+      # Add -outfmt 11 to list of options so that it outputs a blast archive
+      @options = options.to_s
+      @options += " -html"
+    end
+
+    # Run blast everytime it is called. Returns the success
+    # status - true, or false.
+    def run!
+      @result, @error, status = execute(command)
+
+      status == 0 and return @success = true
+
+      if status == 1
+        message = @error.each{|l| l.match(ERROR_LINE) and break Regexp.last_match[1]}
+        message = message || @error
+        @error  = [400,  message]
+      else
+        @error = [500, @error]
+      end
+
+      false
+    end
+
+    # The command that will be executed.
+    def command
+      @command ||= "#@method -db '#@databases' -query '#{@qfile.path}' #@options"
+    end
+
+    # Return success status.
+    def success?
+      @success
+    end
+
+    private
+
+    # Execute a command and return its stdout, stderr, and exit status.
+    def execute(command)
+      rfile = Tempfile.new('sequenceserver_result')
+      efile = Tempfile.new('sequenceserver_error')
+      [rfile, efile].each {|file| file.close}
+
+      system("#{command} > #{rfile.path} 2> #{efile.path}")
+      status = $?.exitstatus
+
+      return File.readlines(rfile.path), File.readlines(efile.path), status
+    ensure
+      rfile.unlink
+      efile.unlink
+    end
+  end
+end

data/lib/sequenceserver/customisation.rb

@@ -0,0 +1,60 @@
+module SequenceServer
+  module Customisation
+    ## When not commented out, this method is used to take a
+    ## sequence ID, and return a hyperlink that
+    ## replaces the hit in the BLAST output.
+    ##
+    ## Return the hyperlink to link to, or nil
+    ## to not not include a hyperlink.
+    ##
+    ## When this method
+    ## is commented out, the default link is used. The default
+    ## is a link to the full sequence of
+    ## the hit is displayed (if makeblastdb has been run with
+    ## -parse_seqids), or no link at all otherwise.
+    # def construct_custom_sequence_hyperlink(options)
+    #   ## Example:
+    #   ## sequence_id comes in like "psu|MAL13P1.200 | organism=Plasmodium_falciparum_3D7 | product=mitochondrial"
+    #   ## output: "http://apiloc.bio21.unimelb.edu.au/apiloc/gene/MAL13P1.200"
+    #   matches = options[:sequence_id].match(/^\s*psu\|(\S+) /)
+    #   if matches #if the sequence_id conforms to our expectations
+    #     # All is good. Return the hyperlink.
+    #     return "http://apiloc.bio21.unimelb.edu.au/apiloc/gene/#{matches[1]}"
+    #   else
+    #     # Parsing the sequence_id didn't work. Don't include a hyperlink for this
+    #     # sequence_id, but log that there has been a problem.
+    #     settings.log.warn "Unable to parse sequence id `#{options[:sequence_id]}'"
+    #     # Return nil so no hyperlink is generated.
+    #     return nil
+    #   end
+    # end
+
+    ## Much like construct_custom_sequence_hyperlink, except
+    ## instead of just a hyperlink being defined, the whole
+    ## line as it appears in the blast results is generated.
+    ##
+    ## This is a therefore more flexible setup than is possible
+    ## with construct_custom_sequence_hyperlink, because doing
+    ## things such as adding two hyperlinks for the one hit
+    ## are possible.
+    ##
+    ## When this method is commented out, the behaviour is that
+    ## the construct_custom_sequence_hyperlink method is used,
+    ## or failing that the default method of that is used.
+    # def construct_custom_sequence_hyperlinking_line(options)
+    #   matches = options[:sequence_id].match(/^\s*psu\|(\S+) /)
+    #   if matches #if the sequence_id conforms to our expectations
+    #     # All is good. Return the hyperlink.
+    #     link1 = "http://apiloc.bio21.unimelb.edu.au/apiloc/gene/#{matches[1]}"
+    #     link2 = "http://google.com/?q=#{matches[1]}"
+    #     return "<a href='#{link1}'>ApiLoc page</a>, <a href='#{link2}'>Google search</a>"
+    #   else
+    #     # Parsing the sequence_id didn't work. Don't include a hyperlink for this
+    #     # sequence_id, but log that there has been a problem.
+    #     settings.log.warn "Unable to parse sequence id `#{options[:sequence_id]}'"
+    #     # Return nil so no hyperlink is generated.
+    #     return nil
+    #   end
+    # end
+  end
+end

data/lib/sequenceserver/database.rb

@@ -0,0 +1,29 @@
+require 'digest/md5'
+
+module SequenceServer
+  class Database < Struct.new("Database", :name, :title, :type)
+    def to_s
+      "#{type}: #{title} #{name}"
+    end
+
+    # Its not very meaningful to compare Database objects, however,
+    # we still add the 'spaceship' operator to be able to sort the
+    # databases by 'title', or 'name' for better visual presentation.
+    # 
+    # We use 'title' for comparison, while relying on 'name' as fallback.
+    #
+    # Trying to sort a list of dbs with 'title' set only for some of them
+    # will obviously produce unpredictable sorting order.
+    def <=>(other)
+      if self.title and other.title
+        self.title <=> other.title
+      else
+        self.name <=> other.name
+      end
+    end
+
+    def hash
+      @hash ||= Digest::MD5.hexdigest(self.name)
+    end
+  end
+end

data/lib/sequenceserver/database_formatter.rb

@@ -0,0 +1,190 @@
+# copyright yannick . wurm at unil . ch
+# Finds files, reads first char. if its '>', read 500 lines. Guess sequence type, ask user for title to format as blast database.
+
+# TODO: bring it under SequenceServer namespace
+# TODO: move the file to a 'command/' sub-directory (probably makes more sense if we have several subcommands)
+# TODO: needs more love (read refactoring) overall
+
+require 'ptools' # for File.binary?(file)
+require 'find'
+require 'logger'
+require 'optparse'
+require 'sequenceserver'
+require 'sequenceserver/helpers.rb'
+require 'sequenceserver/sequencehelpers.rb'
+
+LOG = Logger.new(STDOUT)
+LOG.level = Logger::INFO
+
+class DatabaseFormatter
+    include SequenceServer
+	include Helpers
+    include SystemHelpers
+    include SequenceHelpers
+    
+    attr_accessor :db_path
+
+    def initialize(db_path = nil)
+      @app = SequenceServer::App
+      @app.config = @app.parse_config
+      @app.binaries = @app.scan_blast_executables(@app.bin).freeze
+
+      @db_path = (db_path or @app.database)
+    end
+
+    def format_databases
+      unless File.directory?(db_path)
+        LOG.fatal("Database directory #{db_path} not found. See './database_formatter --help' for instructions.")
+        exit
+      end
+
+        formatted_dbs = %x|#{@app.binaries['blastdbcmd']} -recursive -list #{db_path} -list_outfmt "%f" 2>&1|.split("\n")
+        commands = []
+        Find.find(db_path) do |file|
+            LOG.debug("Assessing file #{file}..")
+            if File.directory?(file)
+              LOG.debug("Ignoring file #{file} since it is a directory")
+              next
+            end
+            if formatted_dbs.include?(file)
+              LOG.debug("Ignoring file #{file} since it is already a blast database")
+              next
+            end
+            if File.binary?(file)
+              LOG.debug("Ignoring file #{file} since it is a binary file, not plaintext as FASTA files are")
+              next
+            end
+
+                if probably_fasta?(file)
+                    LOG.info("Found #{file}")
+                    ## guess whether protein or nucleotide based on first 500 lines
+                    first_lines = ''
+                    File.open(file, 'r') do |file_stream|
+                        file_stream.each do |line|
+                            first_lines += line
+                            break if file_stream.lineno == 500
+                        end
+                    end
+                    begin
+                        sequence_type = type_of_sequences(first_lines) # returns :protein or :nucleotide
+                    rescue
+                        LOG.warn("Unable to guess sequence type for #{file}. Skipping") 
+                    end
+                    if [ :protein, :nucleotide ].include?(sequence_type)
+                        command = ask_make_db_command(file, sequence_type)
+                        unless command.nil?
+                            commands.push(command)
+                        end
+                    else 
+                        LOG.warn("Unable to guess sequence type for #{file}. Skipping") 
+                    end
+                else
+                  LOG.debug("Ignoring file #{file} since it was not judged to be a FASTA file.")
+                end
+        end
+        LOG.info("Will now create DBs")
+        if commands.empty?
+          puts "", "#{db_path} does not contain any unformatted database."
+          exit
+        end
+        commands.each do |command|
+			LOG.info("Will run: " + command.to_s)
+            system(command)
+        end
+        LOG.info("Done formatting databases. ")
+        db_table(db_path)
+    end
+
+    def db_table(db_path)
+        LOG.info("Summary of formatted blast databases:\n")
+        output = %x|#{@app.binaries['blastdbcmd']} -recursive -list #{db_path} -list_outfmt "%p %f %t" &2>1 |
+        LOG.info(output)
+    end
+
+    def probably_fasta?(file)
+		return FALSE if File.zero?(file)
+        File.open(file, 'r') do |file_stream|
+            first_line = file_stream.readline
+            if first_line.slice(0,1) == '>'
+                return TRUE
+            else 
+                return FALSE
+            end
+        end
+    end
+
+
+    # returns command than needs to be run to make db
+    def ask_make_db_command(file, type)
+        LOG.info("FASTA file: #{file}")
+        LOG.info("Fasta type: " + type.to_s)
+        
+        response = ''
+        until response.match(/^[yn]$/i) do
+            LOG.info("Proceed? [y/n]: ")
+            response = STDIN.gets.chomp
+        end
+
+        if response.match(/y/i)
+            LOG.info("Enter a database title (or will use '#{File.basename(file)}'")
+            title = STDIN.gets.chomp
+			title.gsub!('"', "'")
+            title = File.basename(file)  if title.empty?
+            
+            return make_db_command(file,type,title)
+        end
+    end
+
+    def make_db_command(file,type, title)
+        LOG.info("Will make #{type.to_s} database from #{file} with #{title}")
+        command = %|#{@app.binaries['makeblastdb']} -in #{file} -dbtype #{ type.to_s.slice(0,4)} -title "#{title}" -parse_seqids|
+        LOG.info("Returning: #{command}")
+        return(command)
+    end
+end
+
+OptionParser.new do |opts|
+  opts.banner =<<BANNER
+
+SUMMARY
+
+  prepare BLAST databases for SequenceServer
+
+USAGE
+
+  sequenceserver format-databases [--verbose] [blast_database_directory]
+
+  Example:
+
+    $ sequenceserver format-databases ~/db  # explicitly specify a database directory
+    $ sequenceserver format-databases      # use the database directory in config.yml
+
+DESCRIPTION
+
+  Recursively scan the given 'blast_database_directory' for BLAST databases and
+  formats them for use with SequenceServer.
+
+  It automagically detects the database type, and ignores non-db files and
+  pre-formatted databases. The 'parse_seqids' makeblastdb options is used.
+
+  'blast_database_directory' can be passed as a command line parameter or
+  through a configuration file by setting the 'database' key (the same option
+  used by SequenceServer). Configuration file will be checked only if the
+  command line parameter is missing.
+
+OPTIONS
+
+BANNER
+
+  opts.on_tail('-h', '--help', 'Show this message') do
+    puts opts
+    exit
+  end
+
+  opts.on('-v', '--verbose', 'Print lots of output') do
+    LOG.level = Logger::DEBUG
+  end
+end.parse!
+
+app = DatabaseFormatter.new(ARGV[0])
+app.format_databases

data/lib/sequenceserver/helpers.rb

@@ -0,0 +1,136 @@
+require 'sequenceserver/database'
+
+module SequenceServer
+  module Helpers
+    module SystemHelpers
+      # Scan the given directory for blast executables. Passing `nil` scans the
+      # system `PATH`.
+      # ---
+      # Arguments:
+      # * bin(String) - absolute path to the directory containing blast binaries
+      # ---
+      # Returns:
+      # * a hash of blast methods, and their corresponding absolute path
+      # ---
+      # Raises:
+      # * IOError - if the executables can't be found
+      #
+      #   > scan_blast_executables('/home/yeban/bin')
+      #   =>  { "blastx"=>"/home/yeban/bin/blastx",
+      #         "blastn"=>"/home/yeban/bin/blastn",
+      #         ...
+      #       }
+      def scan_blast_executables(bin)
+        if bin and not File.directory?(bin)
+          raise IOError, "Could not find '#{bin}' defined in config.yml."
+        end
+
+        binaries = {}
+        %w|blastn blastp blastx tblastn tblastx blastdbcmd makeblastdb blast_formatter|.each do |method|
+          path = File.join(bin, method) rescue method
+          if command?(path)
+            binaries[method] = path
+          else
+            blasturl = 'http://www.ncbi.nlm.nih.gov/blast/Blast.cgi?CMD=Web&PAGE_TYPE=BlastDocs&DOC_TYPE=Download'
+            raise IOError, "Could not find blast binaries." +
+            "\n\nYou may need to download BLAST+ from #{blasturl}." +
+            " And/or edit #{settings.config_file} to indicate the location of BLAST+ binaries."
+          end
+        end
+
+        #LOG.info("Config bin dir:          #{bin}")
+        binaries
+      end
+
+      # Scan the given directory (including subdirectory) for blast databases.
+      # ---
+      # Arguments:
+      # * db_root(String) - absolute path to the blast databases
+      # ---
+      # Returns:
+      # * a hash of sorted blast databases grouped by database type:
+      # protein, or nucleotide
+      # ---
+      # Raises:
+      # * IOError - if no database can be found
+      #
+      #   > scan_blast_db('/home/yeban/blast_db')
+      #   => { "protein" => [], "nucleotide" => [] }
+      def scan_blast_db(db_root, blastdbcmd = 'blastdbcmd')
+        raise IOError, "Database directory doesn't exist: #{db_root}" unless File.directory?( db_root )
+
+        find_dbs_command = %|#{blastdbcmd} -recursive -list #{db_root} -list_outfmt "%p %f %t" 2>&1|
+
+        begin
+          db_list = %x|#{find_dbs_command}|
+          if db_list.empty?
+            raise IOError, "No formatted blast databases found in '#{ db_root }'."
+          end
+        rescue => e
+          puts '', e.to_s
+
+          print "Do you want to format your blast databases now? [Y/n]: "
+          choice = gets.chomp[0,1].downcase
+
+          unless choice == 'n'
+            database_formatter = File.join(settings.root, 'database_formatter.rb')
+            system("#{database_formatter} #{db_root}")
+            retry
+          else
+            raise # let the caller decide what to do if database discovery fails
+          end
+        end
+
+        if db_list.match(/BLAST Database error/)
+          raise IOError, "Error parsing blast databases.\n" + "Tried: '#{find_dbs_command}'\n"+
+            "It crashed with the following error: '#{db_list}'\n" +
+            "Try reformatting databases using makeblastdb.\n"
+        end
+
+        db = {}
+
+        db_list.each_line do |line|
+          next if line.empty?  # required for BLAST+ 2.2.22
+          type, name, *title =  line.split(' ') 
+          type = type.downcase.intern
+          name = name.freeze
+          title = title.join(' ').freeze
+
+          # skip past all but alias file of a NCBI multi-part BLAST database
+          if multipart_database_name?(name)
+            log.info(%|Found a multi-part database volume at #{name} - ignoring it.|)
+            next
+          end
+
+          #LOG.info("Found #{type} database: #{title} at #{name}")
+          database = Database.new(name, title, type)
+          db[database.hash] = database
+        end
+
+        db 
+      end
+
+      private
+
+      # check if the given command exists and is executable
+      # returns True if all is good.
+      def command?(command)
+        system("which #{command} > /dev/null 2>&1")
+      end
+
+      # Returns true if the database name appears to be a multi-part database name.
+      #
+      # e.g.
+      # /home/ben/pd.ben/sequenceserver/db/nr.00 => yes
+      # /home/ben/pd.ben/sequenceserver/db/nr => no
+      # /home/ben/pd.ben/sequenceserver/db/img3.5.finished.faa.01 => yes
+      def multipart_database_name?(db_name)
+        !(db_name.match(/.+\/\S+\d{2}$/).nil?)
+      end
+    end
+
+    def self.included(klass)
+      klass.extend SystemHelpers
+    end
+  end
+end