sequenceserver 0.8.9 → 1.0.0.pre.1

data/lib/sequenceserver/database.rb

@@ -1,29 +1,195 @@
+require 'find'
 require 'digest/md5'
+require 'forwardable'
+
+require 'sequenceserver/sequence'
 
 module SequenceServer
-  class Database < Struct.new("Database", :name, :title, :type)
-    def to_s
-      "#{type}: #{title} #{name}"
-    end
 
-    # Its not very meaningful to compare Database objects, however,
-    # we still add the 'spaceship' operator to be able to sort the
-    # databases by 'title', or 'name' for better visual presentation.
-    # 
-    # We use 'title' for comparison, while relying on 'name' as fallback.
-    #
-    # Trying to sort a list of dbs with 'title' set only for some of them
-    # will obviously produce unpredictable sorting order.
-    def <=>(other)
-      if self.title and other.title
-        self.title <=> other.title
-      else
-        self.name <=> other.name
+  # Captures a directory containing FASTA files and BLAST databases.
+  #
+  # Formatting a FASTA for use with BLAST+ will create 3 or 6 files,
+  # collectively referred to as a BLAST database.
+  #
+  # It is important that formatted BLAST database files have the same dirname and
+  # basename as the source FASTA for SequenceServer to be able to tell formatted
+  # FASTA from unformatted. And that FASTA files be formatted with `parse_seqids`
+  # option of `makeblastdb` for sequence retrieval to work.
+  #
+  # SequenceServer will always place BLAST database files alongside input FASTA,
+  # and use `parse_seqids` option of `makeblastdb` to format databases.
+  class Database < Struct.new(:name, :title, :type, :nsequences, :ncharacters, :updated_on)
+
+    class << self
+
+      include Enumerable
+
+      extend Forwardable
+
+      def_delegators SequenceServer, :config, :logger
+
+      def collection
+        @collection ||= {}
+      end
+
+      private :collection
+
+      def <<(database)
+        collection[database.id] = database
+      end
+
+      def [](ids)
+        ids = Array ids
+        collection.values_at(*ids)
+      end
+
+      def ids
+        collection.keys
+      end
+
+      def all
+        collection.values
+      end
+
+      def each(&block)
+        all.each(&block)
+      end
+
+      def include?(path)
+        collection.include? Digest::MD5.hexdigest path
+      end
+
+      def group_by(&block)
+        all.group_by(&block)
+      end
+
+      # Intended to be used only for testing.
+      def first
+        all.first
+      end
+
+      # Recurisvely scan `database_dir` for blast databases.
+      def scan_databases_dir
+        database_dir = config[:database_dir]
+        list = %x|blastdbcmd -recursive -list #{database_dir} -list_outfmt "%f	%t	%p	%n	%l	%d" 2>&1|
+        list.each_line do |line|
+          name = line.split('	')[0]
+          next if multipart_database_name?(name)
+          self << Database.new(*line.split('	'))
+        end
+      end
+
+      # Recursively scan `database_dir` for un-formatted FASTA and format them
+      # for use with BLAST+.
+      def make_blast_databases
+        unformatted_fastas.each do |file, sequence_type|
+          make_blast_database(file, sequence_type)
+        end
+      end
+
+      # Returns an Array of FASTA files that may require formatting, and the type
+      # of sequence contained in each FASTA.
+      #
+      #   > unformatted_fastas
+      #   => [['/foo/bar.fasta', :nulceotide], ...]
+      def unformatted_fastas
+        list = []
+        database_dir = config[:database_dir]
+        Find.find database_dir do |file|
+          next if File.directory?(file)
+          next if Database.include? file
+          if probably_fasta? file
+            sequence_type = guess_sequence_type_in_fasta file
+            if [:protein, :nucleotide].include?(sequence_type)
+              list << [file, sequence_type]
+            end
+          end
+        end
+        list
+      end
+
+      # Create BLAST database, given FASTA file and sequence type in FASTA file.
+      def make_blast_database(file, type)
+        puts "FASTA file: #{file}"
+        puts "FASTA type: #{type}"
+
+        print "Proceed? [y/n] (Default: y): "
+        response = STDIN.gets.to_s.strip
+
+        unless response.match(/n/i)
+          default_title = make_db_title(File.basename(file))
+          print "Enter a database title or will use '#{default_title}': "
+          title = STDIN.gets.to_s
+          title = default_title if title.strip.empty?
+
+          `makeblastdb -parse_seqids -hash_index \
+            -in #{file} -dbtype #{type.to_s.slice(0,4)} -title "#{title}"`
+        end
+      end
+
+      # Returns true if the database name appears to be a multi-part database name.
+      #
+      # e.g.
+      # /home/ben/pd.ben/sequenceserver/db/nr.00 => yes
+      # /home/ben/pd.ben/sequenceserver/db/nr => no
+      # /home/ben/pd.ben/sequenceserver/db/img3.5.finished.faa.01 => yes
+      def multipart_database_name?(db_name)
+        !(db_name.match(/.+\/\S+\d{2}$/).nil?)
+      end
+
+      # Returns true if first character of the file is '>'.
+      def probably_fasta?(file)
+        File.read(file, 1) == '>'
+      end
+
+      # Suggests improved titles when generating database names from files
+      # for improved apperance and readability in web interface.
+      # For example:
+      # Cobs1.4.proteins.fasta -> Cobs 1.4 proteins
+      # S_invicta.xx.2.5.small.nucl.fa -> S invicta xx 2.5 small nucl
+      def make_db_title(db_name)
+        db_name.gsub!('"', "'")
+        # removes .fasta like extension names
+        db_name.gsub!(File.extname(db_name), '')
+        # replaces _ with ' ',
+        db_name.gsub!(/(_)/, ' ')
+        # replaces '.' with ' ' when no numbers are on either side,
+        db_name.gsub!(/(?<![0-9])\.(?![0-9])/, ' ')
+        # preserves version numbers
+        db_name.gsub!(/\W*(\d+([.-]\d+)+)\W*/, ' \1 ')
+        db_name
+      end
+
+      # Guess whether FASTA file contains protein or nucleotide sequences based
+      # on first 32768 characters.
+      #
+      # NOTE: 2^15 == 32786. Approximately 546 lines, assuming 60 characters on
+      # each line.
+      def guess_sequence_type_in_fasta(file)
+        sample = File.read(file, 32768)
+        sequences = sample.split(/^>.+$/).delete_if { |seq| seq.empty? }
+        sequence_types = sequences.map {|seq| Sequence.guess_type(seq)}.uniq.compact
+        (sequence_types.length == 1) && sequence_types.first
       end
     end
 
-    def hash
-      @hash ||= Digest::MD5.hexdigest(self.name)
+    def initialize(*args)
+      args[2].downcase!   # database type
+      args.each(&:freeze)
+      super
+
+      @id = Digest::MD5.hexdigest args.first
+    end
+
+    attr_reader :id
+
+    def include?(accession)
+      out = `blastdbcmd -entry '#{accession}' -db #{name} 2> /dev/null`
+      not out.empty?
+    end
+
+    def to_s
+      "#{type}: #{title} #{name}"
     end
   end
 end

data/lib/sequenceserver/links.rb

@@ -0,0 +1,114 @@
+module SequenceServer
+  # Module to contain methods for generating sequence retrieval links.
+  module Links
+    require 'erb'
+
+    # Provide a method to URL encode _query parameters_. See [1].
+    include ERB::Util
+    #
+    alias encode url_encode
+
+    NCBI_ID_PATTERN = /gi\|(\d+)\|/
+
+    # Your custom method should have following pattern:
+    #
+    # Input
+    # -----
+    # sequence_id: Array of sequence ids
+    #
+    # Return
+    # ------
+    # The return value should be a Hash:
+    #
+    # {
+    #   # Required. Display title.
+    #   :title => "title",
+    #
+    #   # Required. Generated url.
+    #   :url => url,
+    #
+    #   # Optional. Left-right order in which the link should appear.
+    #   :order => num,
+    #
+    #   # Optional. Classes, if any, to apply to the link.
+    #   :class => "class1 class2",
+    #
+    #   # Optional. Class name of a FontAwesome icon to use.
+    #   :icon => "fa-icon-class"
+    # }
+    #
+    # If no url could be generated, return nil.
+    #
+    # Helper methods
+    # --------------
+    #
+    # Following helper methods are available to help with link generation.
+    #
+    #   encode:
+    #     URL encode query params.
+    #
+    #     Don't use this function to encode the entire URL. Only params.
+    #
+    #     e.g:
+    #         sequence_id = encode sequence_id
+    #         url = "http://www.ncbi.nlm.nih.gov/nucleotide/#{sequence_id}"
+    #
+    #   querydb:
+    #     Returns an array of databases that were used for BLASTing.
+    #
+    #   which_blastdb:
+    #     Returns the database from which the given hit came from.
+    #
+    #     e.g:
+    #
+    #         hit_database = which_blastdb sequence_id
+    #
+    # Examples:
+    # ---------
+    # See methods provided by default for an example implementation.
+
+    def sequence_viewer(sequence_id)
+      sequence_id  = encode sequence_id
+      database_ids = encode querydb.map(&:id).join(' ')
+      url = "get_sequence/?sequence_ids=#{sequence_id}" \
+            "&database_ids=#{database_ids}"
+
+      {
+        :order => 0,
+        :url   => url,
+        :title => 'Sequence',
+        :class => 'view-sequence',
+        :icon  => 'fa-eye'
+      }
+    end
+
+    def fasta_download(sequence_id)
+      sequence_id  = encode sequence_id
+      database_ids = encode querydb.map(&:id).join(' ')
+      url = "get_sequence/?sequence_ids=#{sequence_id}" \
+            "&database_ids=#{database_ids}&download=fasta"
+
+      {
+        :order => 1,
+        :title => 'FASTA',
+        :url   => url,
+        :icon  => 'fa-download'
+      }
+    end
+
+    def ncbi(sequence_id)
+      return nil unless sequence_id.match(NCBI_ID_PATTERN)
+      ncbi_id = Regexp.last_match[1]
+      ncbi_id = encode ncbi_id
+      url = "http://www.ncbi.nlm.nih.gov/#{querydb.first.typ}/#{ncbi_id}"
+      {
+        :order => 2,
+        :title => 'View on NCBI',
+        :url   => url,
+        :icon  => 'fa-external-link'
+      }
+    end
+  end
+end
+
+# [1]: https://stackoverflow.com/questions/2824126/whats-the-difference-between-uri-escape-and-cgi-escape

data/lib/sequenceserver/logger.rb

@@ -0,0 +1,27 @@
+require 'logger'
+
+module SequenceServer
+
+  class Logger < Logger
+
+    def initialize(dev, verbose = false)
+      super dev
+      self.level     = verbose ? DEBUG : INFO
+      self.formatter = Formatter.new
+    end
+
+    # We change Logging format so that it is consistent with Sinatra's
+    class Formatter < Formatter
+
+      Format = "[%s] %s  %s\n"
+
+      def initialize
+        self.datetime_format = "%Y-%m-%d %H:%M:%S"
+      end
+
+      def call(severity, time, progname, msg)
+        Format % [format_datetime(time), severity, msg2str(msg)]
+      end
+    end
+  end
+end

data/lib/sequenceserver/sequence.rb

@@ -0,0 +1,141 @@
+require 'forwardable'
+
+module SequenceServer
+
+  # Provides simple sequence processing utilities via class methods. Instance
+  # of the class serves as a simple data object to captures sequences fetched
+  # from BLAST databases.
+  #
+  # NOTE:
+  #   What all do we need to consistently construct FASTA from `blastdbcmd's`
+  #   output?
+  #
+  #   It would seem rather straightforward. But it's not.
+  #
+  #   FASTA format:
+  #
+  #     >defline
+  #     actual sequence
+  #
+  #   where,
+  #
+  #     defline = >id title
+  #
+  #   ID of a sequence fetched from nr database should look like this:
+  #
+  #                  sequence id    -> self.seqid
+  #                  -------------
+  #                     accession   -> self.accession
+  #                     ----------
+  #     gi|322796550|gb|EFZ19024.1| -> self.id
+  #        ---------
+  #        gi number                -> self.gi
+  #
+  #   while for local databases, the id should be the exact same, as in the
+  #   original FASTA file:
+  #
+  #     SI2.2.0_06267 -> self.id == self.seqid == self.accession.
+  class Sequence < Struct.new(:gi, :seqid, :accession, :title, :value)
+
+    class << self
+
+      extend Forwardable
+
+      # Derive `logger` from SequenceServer module.
+      def_delegators SequenceServer, :logger
+
+      # Disable using `Sequence.new`. Should use `Sequence.from_blastdb`
+      # instead.
+      private :new
+
+      # Returns an Array of `Sequence` objects each capturing a sequence
+      # fetched from BLAST database.
+      def from_blastdb(accessions, database_ids)
+        accessions = Array accessions
+        database_ids = Array database_ids
+
+        accessions = accessions.join(',')
+        database_names = Database[database_ids].map(&:name).join(' ')
+
+        # Output of the command will be five columns TSV.
+        command = "blastdbcmd -outfmt '%g	%i	%a	%t	%s'" \
+                  " -db '#{database_names}' -entry '#{accessions}'"
+
+        logger.debug("Executing: #{command}")
+
+        # Not interested in stderr.
+        `#{command} 2> /dev/null`.
+          each_line.map {|line| new(*line.chomp.split('	'))}
+      end
+
+      # Strips all non-letter characters. If less than 10 useable characters
+      # return `nil`. If at least 90% is ACGTU, returns `:nucleotide`, else
+      # `:protein`.
+      def guess_type(sequence)
+        # Clean the sequence: first remove non-letter characters, then
+        # ambiguous characters.
+        cleaned_sequence = sequence.gsub(/[^A-Z]/i, '').gsub(/[NX]/i, '')
+
+        return if cleaned_sequence.length < 10 # conservative
+
+        # Count putative NA in the sequence.
+        na_count = 0
+        composition = composition(cleaned_sequence)
+        composition.each do |character, count|
+          na_count = na_count + count if character.match(/[ACGTU]/i)
+        end
+
+        na_count > (0.9 * cleaned_sequence.length) ? :nucleotide : :protein
+      end
+
+      # Copied from BioRuby's `Bio::Sequence` class.
+      #
+      # > composition("asdfasdfffffasdf")
+      # => {"a"=>3, "d"=>3, "f"=>7, "s"=>3}
+      def composition(sequence_string)
+        count = Hash.new(0)
+        sequence_string.scan(/./) do |x|
+          count[x] += 1
+        end
+        count
+      end
+    end
+
+    def initialize(*args)
+      args[0] = nil if args[0] == 'N/A'
+      super
+    end
+
+    # Returns FASTA sequence id.
+    def id
+      (gi ? ['gi', gi, seqid] : [seqid]).join('|')
+    end
+
+    # Returns length of the sequence.
+    def length
+      value.length
+    end
+
+    # Returns sequence value.
+    def to_s
+      value
+    end
+
+    def info
+      {:value => value, :id => id, :title => title}
+    end
+
+    # Returns FASTA formatted sequence.
+    def fasta
+      chars = 60
+      lines = (length / chars.to_f).ceil
+      defline  = ">#{id} #{title}"
+      seqlines = (1..lines).map {|i| to_s[chars * (i - 1), chars]}
+      [defline].concat(seqlines).join("\n")
+    end
+  end
+end
+
+# References
+# ----------
+# [1]: http://blast.ncbi.nlm.nih.gov/blastcgihelp.shtml