sequenceserver 1.1.0.beta12 → 2.0.0.beta1

data/lib/sequenceserver/api_errors.rb

@@ -18,7 +18,9 @@ module SequenceServer
       'The requested job could not be found'
     end
 
-    undef_method :backtrace
+    def more_info
+      ''
+    end
   end
 
   # Errors caused due to incorrect user input.

data/lib/sequenceserver/blast/hit.rb

@@ -1,12 +1,9 @@
 module SequenceServer
   # Define BLAST::Hit.
   module BLAST
-    # Hit Object to store all the hits per Query. HSPs per hit should be sorted
-    # in ascending order of evalue.
+    # Hit object to store all the hits per Query.
     Hit = Struct.new(:query, :number, :id, :accession, :title,
                      :length, :sciname, :qcovs, :hsps) do
-      include Links
-
       def initialize(*args)
         args[1] = args[1].to_i
         args[4] = '' if args[4] == 'No definition line'
@@ -16,29 +13,45 @@ module SequenceServer
         super
       end
 
-      # Hit's score is the sum of score of all HSPs.
-      def score
-        hsps.map(&:score).reduce(:+)
+      # This gets called when #to_json is called on report object in routes. We
+      # cannot use the to_json method provided by Struct class because what we
+      # want to send to the browser differs from the attributes declared with
+      # Struct class. Some of these are derived data such as score, identity,
+      # custom links. While some attributes are necessary for internal
+      # representation.
+      def to_json(*args)
+        # List all attributes that we want to send to the browser.
+        properties = %i[number id accession title length score identity
+                        qcovs sciname evalue hsps links]
+        properties.inject({}) { |h, k| h[k] = send(k); h }.to_json(*args)
       end
 
-      # Hit's identity is the sum of identity of all
-      # HSPs divided by sum of length of all HSPs
-      # (expressed as percentagge).
-      def identity
-        hsps.map(&:identity).reduce(:+) * 100 / hsps.map(&:length).reduce(:+)
-      end
+      ###
+      # Link generator functionality.
+      ###
+
+      # Include the Links module.
+      include Links
 
+      # Links returns a list of Hashes that can be easily turned into an href
+      # in the client. These are derived by calling link generators, that is,
+      # instance methods of the Links module.
       def links
         links = Links.instance_methods.map { |m| send m }
         links.compact!
         links.sort_by { |link| [link[:order], link[:title]] }
       end
 
+      # Returns the database type (nucleotide or protein).
+      def dbtype
+        report.dbtype
+      end
+
       # Returns a list of databases that contain this hit.
       #
       # e.g., whichdb('SI_2.2.23') => [<Database: ...>, ...]
       def whichdb
-        querydb.select { |db| db.include? id }
+        report.querydb.select { |db| db.include? id }
       end
 
       # Returns tuple of tuple indicating start and end coordinates of matched
@@ -52,26 +65,33 @@ module SequenceServer
         [[qstart_min, qend_max], [sstart_min, send_max]]
       end
 
-      # NOTE: Evalue of a hit is meaningless. This is here for code that needs
-      # minimum evalue of all HSPs.
-      def evalue
-        hsps.first.evalue
-      end
+      ###
+      # Score, identity, and evalue attributes below are used in tabular summary
+      # of hits in the HTML report. At some point we should move these to the
+      # client.
+      ###
 
-      def to_json(*args)
-        %i[number id accession title length score identity qcovs
-         sciname evalue hsps links].inject({}) { |h, k|
-          h[k] = send(k)
-          h
-        }.to_json(*args)
+      # Returns the sum of scores of all HSPs.
+      def score
+        hsps.map(&:score).reduce(:+)
       end
 
-      private
+      # Returns the sum of identity of all HSPs divided by sum of length of all
+      # HSPs (expressed as percentage).
+      def identity
+        hsps.map(&:identity).reduce(:+) * 100 / hsps.map(&:length).reduce(:+)
+      end
 
-      def querydb
-        report.querydb
+      # Returns the minimum evalue of all HSPs of the Hit. This is shown in the
+      # tabular overview of hits in the HTML report.
+      def evalue
+        hsps.first.evalue
       end
 
+      private
+
+      # Returns the report object that this hit is a part of. This is used to
+      # access list of databases etc.
       def report
         query.report
       end

data/lib/sequenceserver/blast/job.rb

@@ -6,13 +6,26 @@ module SequenceServer
     # Extends SequenceServer::Job to describe a BLAST job.
     class Job < Job
       def initialize(params)
-        validate params
-        super do
-          @method    = params[:method]
-          @qfile     = store('query.fa', params[:sequence])
-          @databases = Database[params[:databases]]
-          @options   = params[:advanced].to_s.strip + defaults
-          @advanced_params = parse_advanced params[:advanced]
+        if params.key?(:xml)
+          super do
+            @imported_xml_file = File.basename params[:xml]
+            # Copy over the XML file to job directory so that a job dir in
+            # itself is self-contained. This will help with tests among
+            # other things.
+            FileUtils.cp(params[:xml], dir)
+            @advanced_params = {}
+            @databases = []
+            done!
+          end
+        else
+          validate params
+          super do
+            @method    = params[:method]
+            @qfile     = store('query.fa', params[:sequence])
+            @databases = Database[params[:databases]]
+            @options   = params[:advanced].to_s.strip + defaults
+            @advanced_params = parse_advanced params[:advanced]
+          end
         end
       end
 
@@ -22,6 +35,13 @@ module SequenceServer
       # Attributes used by us - should be considered private.
       attr_reader :method, :qfile, :databases, :options
 
+      # :nodoc:
+      # Returns path to the imported xml file if the job was created using the
+      # --import switch. Returns nil otherwise.
+      def imported_xml_file
+        File.join(dir, @imported_xml_file) if @imported_xml_file
+      end
+
       # Returns the command that will be executed. Job super class takes care
       # of actual execution.
       def command
@@ -146,7 +166,7 @@ module SequenceServer
       end
 
       def disallowed_options
-        /-out|-html|-outfmt|-db|-query|-num_threads/i
+        /-out|-html|-outfmt|-db |-query|-num_threads/i
       end
     end
   end

data/lib/sequenceserver/blast/report.rb

@@ -29,18 +29,38 @@ module SequenceServer
         end
       end
 
-      # :nodoc:
       # Attributes parsed out from XML output.
-      attr_reader :program, :program_version
-      attr_reader :queries, :querydb
-      attr_reader :params, :stats
+      attr_reader :program, :program_version, :params, :stats, :queries
+
+      # This is obtained from the job object.
+      attr_reader :querydb
+
+      # Returns database type (nucleotide or protein) used for running BLAST
+      # search. If we ran the BLAST search, this information is available
+      # from Job#databases. For imported XML, this is inferred from
+      # Report#program (i.e., the BLAST algorithm)
+      def dbtype
+        return @dbtype if @dbtype
+        @dbtype = if @querydb.empty?
+                    case program
+                    when /blastn|tblastn|tblastx/
+                      'nucleotide'
+                    when /blastp|blastx/
+                      'protein'
+                    end
+                  else
+                    @querydb.first.type
+                  end
+      end
 
       def to_json
         [:querydb, :program, :program_version, :params, :stats,
          :queries].inject({}) { |h, k|
           h[k] = send(k)
           h
-        }.update(search_id: job.id, submitted_at: job.submitted_at.utc).to_json
+        }.update(search_id: job.id,
+                 submitted_at: job.submitted_at.utc,
+                 imported_xml: !!job.imported_xml_file).to_json
       end
 
       private
@@ -48,12 +68,25 @@ module SequenceServer
       # Generate report.
       def generate
         job.raise!
-        xml_ir = parse_xml File.read(Formatter.run(job, 'xml').file)
-        tsv_ir = parse_tsv File.read(Formatter.run(job, 'custom_tsv').file)
-        extract_program_info xml_ir
-        extract_params xml_ir
-        extract_stats xml_ir
-        extract_queries xml_ir, tsv_ir
+        if job.imported_xml_file
+          xml_ir = parse_xml File.read(job.imported_xml_file)
+          tsv_ir = Hash.new do |h1,k1|
+            h1[k1] = Hash.new do |h2,k2|
+              h2[k2]=['','',[]]
+            end
+          end
+          extract_program_info xml_ir
+          extract_params xml_ir
+          extract_stats xml_ir
+          extract_queries xml_ir, tsv_ir
+        else
+          xml_ir = parse_xml File.read(Formatter.run(job, 'xml').file)
+          tsv_ir = parse_tsv File.read(Formatter.run(job, 'custom_tsv').file )
+          extract_program_info xml_ir
+          extract_params xml_ir
+          extract_stats xml_ir
+          extract_queries xml_ir, tsv_ir
+        end
       end
 
       # Make program name and program name + version available via `program`
@@ -107,10 +140,14 @@ module SequenceServer
       def extract_hits(xml_ir, tsv_ir, query)
         return if xml_ir == ["\n"] # => No hits.
         xml_ir.each do |n|
-          # If hit comes from a non -parse_seqids database, then
-          # we assign id to accession and process hit defline to
-          # obtain id and title.
-          if n[1] =~ /^gnl\|/
+          # If hit comes from a non -parse_seqids database, then id (n[1]) is a
+          # BLAST assigned internal id of the format 'gnl|BL_ORD_ID|serial'. We
+          # assign the id to accession (because we use accession for sequence
+          # retrieval and this id is what blastdbcmd expects for non
+          # -parse_seqids databases) and parse the hit defline to
+          # obtain id and title ourselves (we use id and title
+          # for display purposes).
+          if n[1] =~ /^gnl\|BL_ORD_ID\|\d+/
             n[3] = n[1]
             defline = n[2].split
             n[1] = defline.shift

data/lib/sequenceserver/database.rb

@@ -36,6 +36,19 @@ module SequenceServer
 
     attr_reader :id
 
+    def retrieve(accession, coords = nil)
+      cmd = "blastdbcmd -db #{name} -entry '#{accession}'"
+      if coords
+        cmd << " -range #{coords}"
+      end
+      out, = sys(cmd, path: config[:bin])
+      out.chomp
+    rescue CommandFailed
+      # Command failed beacuse stdout was empty, meaning accession not
+      # present in this database.
+      nil
+    end
+
     def include?(accession)
       cmd = "blastdbcmd -entry '#{accession}' -db #{name}"
       out, = sys(cmd, path: config[:bin])
@@ -103,6 +116,64 @@ module SequenceServer
         collection.values.to_json
       end
 
+      # Retrieve given loci from the databases we have.
+      #
+      # loci to retrieve are specified as a String:
+      #
+      #    "accession_1,accession_2:start-stop,accession_3"
+      #
+      # Return value is a FASTA format String containing sequences in the same
+      # order in which they were requested. If an accession could not be found,
+      # a commented out error message is included in place of the sequence.
+      # Sequences are retrieved from the first database in which the accession
+      # is found. The returned sequences can, thus, be incorrect if accessions
+      # are not unique across all database (admins should make sure of that).
+      def retrieve(loci)
+        # Exit early if loci is nil.
+        return unless loci
+
+        # String -> Array
+        # We may have empty string if loci contains a double comma as a result
+        # of typo (remember - loci is external input). These are eliminated.
+        loci = loci.split(',').delete_if(&:empty?)
+
+        # Each database is searched for each locus. For each locus, search is
+        # terminated on the first database match.
+        # NOTE: This can return incorrect sequence if the sequence ids are
+        # not unique across all databases.
+        seqs = loci.map do |locus|
+          # Get sequence id and coords. coords may be nil. accession can't
+          # be.
+          accession, coords = locus.split(':')
+
+          # Initialise a variable to store retrieved sequence.
+          seq = nil
+
+          # Go over each database looking for this accession.
+          each do |database|
+            # Database lookup  will return a string if given accession is
+            # present in the database, nil otherwise.
+            seq = database.retrieve(accession, coords)
+            # Found a match! Terminate iteration returning the retrieved
+            # sequence.
+            break if seq
+          end
+
+          # If accession was not present in any database, insert an error
+          # message in place of the sequence. The line starts with '#'
+          # and should be ignored by BLAST (not tested).
+          unless seq
+            seq = "# ERROR: #{locus} not found in any database"
+          end
+
+          # Return seq.
+          seq
+        end
+
+        # Array -> String
+        seqs.join("\n")
+      end
+
       # Intended to be used only for testing.
       def first
         all.first

data/lib/sequenceserver/exceptions.rb

@@ -61,21 +61,16 @@ module SequenceServer
     end
   end
 
-  ## BLAST NOT INSTALLED OR NOT COMPATIBLE ##
+  ## BLAST NOT INSTALLED, NOT EXECUTABLE, OR NOT COMPATIBLE ##
 
   # Raised if SequenceServer could not locate NCBI BLAST+ installation on
   # user's system.
   class BLAST_NOT_INSTALLED_OR_NOT_EXECUTABLE < StandardError
     def to_s
-      'BLAST not installed, or is not executable.'
-    end
-  end
-
-  # Raised if SequenceServer could not successfully execute 'blastp -version'
-  # on user's system (see #141).
-  class BLAST_NOT_EXECUTABLE < StandardError
-    def to_s
-      'Error executing BLAST+ binaries.'
+      <<~MSG
+        BLAST+ is either not installed, or there is a problem with the
+        installed version.
+      MSG
     end
   end
 

data/lib/sequenceserver/job.rb

@@ -25,7 +25,7 @@ module SequenceServer
       # Creates and queues a job. Returns created job object.
       def create(params)
         job = BLAST::Job.new(params) # TODO: Dynamic dispatch.
-        SequenceServer.pool.queue { job.run }
+        pool.queue { job.run }
         job
       end
 
@@ -46,6 +46,13 @@ module SequenceServer
         Dir["#{DOTDIR}/**/job.yaml"]
           .map { |f| fetch File.basename File.dirname f }
       end
+
+      private
+
+      # Thread pool used for running BLAST searches.
+      def pool
+        @pool ||= Pool.new(SequenceServer.config[:num_threads])
+      end
     end
 
     include FileUtils

data/lib/sequenceserver/links.rb

@@ -9,8 +9,10 @@ module SequenceServer
 
     NCBI_ID_PATTERN    = /gi\|(\d+)\|/
     UNIPROT_ID_PATTERN = /sp\|(\w+)\|/
+    PFAM_ID_PATTERN = /(PF\d{5}\.?\d*)/
+    RFAM_ID_PATTERN = /(RF\d{5})/
 
-    # Link generators return a Hash like below.
+    # Link generators are methods that return a Hash as defined below.
     #
     # {
     #   # Required. Display title.
@@ -45,25 +47,29 @@ module SequenceServer
     #         sequence_id = encode sequence_id
     #         url = "http://www.ncbi.nlm.nih.gov/nucleotide/#{sequence_id}"
     #
-    #   querydb:
-    #     Returns an array of databases that were used for BLASTing.
+    #   dbtype:
+    #     Returns the database type (nucleotide or protein) that was used for
+    #     BLAST search.
     #
     #   whichdb:
-    #     Returns the database from which the given hit came from.
-    #
-    #     e.g:
-    #
-    #         hit_database = whichdb
-    #
-    # Examples:
-    # ---------
-    # See methods provided by default for an example implementation.
+    #     Returns the databases from which the hit could have originated. To
+    #     ensure that one and the correct database is returned, ensure that
+    #     your sequence ids are unique across different FASTA files.
+    #     NOTE: This method is slow.
+    #
+    #   coordinates:
+    #     Returns min alignment start and max alignment end coordinates for
+    #     query and hit sequences.
+    #
+    #     e.g.,
+    #     query_coords = coordinates[0]
+    #     hit_coords = coordinates[1]
 
     def ncbi
-      return nil unless id.match(NCBI_ID_PATTERN)
+      return nil unless id.match(NCBI_ID_PATTERN) or title.match(NCBI_ID_PATTERN)
       ncbi_id = Regexp.last_match[1]
       ncbi_id = encode ncbi_id
-      url = "http://www.ncbi.nlm.nih.gov/#{querydb.first.type}/#{ncbi_id}"
+      url = "https://www.ncbi.nlm.nih.gov/#{dbtype}/#{ncbi_id}"
       {
         order: 2,
         title: 'NCBI',
@@ -73,13 +79,39 @@ module SequenceServer
     end
 
     def uniprot
-      return nil unless id.match(UNIPROT_ID_PATTERN)
+      return nil unless id.match(UNIPROT_ID_PATTERN) or title.match(UNIPROT_ID_PATTERN)
       uniprot_id = Regexp.last_match[1]
       uniprot_id = encode uniprot_id
-      url = "http://www.uniprot.org/uniprot/#{uniprot_id}"
+      url = "https://www.uniprot.org/uniprot/#{uniprot_id}"
+      {
+        order: 2,
+        title: 'UniProt',
+        url:   url,
+        icon:  'fa-external-link'
+      }
+    end
+ 
+    def pfam
+      return nil unless id.match(PFAM_ID_PATTERN) or title.match(PFAM_ID_PATTERN)
+      pfam_id = Regexp.last_match[1]
+      pfam_id = encode pfam_id
+      url = "https://pfam.xfam.org/family/#{pfam_id}"
+      {
+        order: 2,
+        title: 'Pfam',
+        url:   url,
+        icon:  'fa-external-link'
+      }
+    end
+
+    def rfam
+      return nil unless id.match(RFAM_ID_PATTERN) or title.match(RFAM_ID_PATTERN)
+      rfam_id = Regexp.last_match[1]
+      rfam_id = encode rfam_id
+      url = "https://rfam.xfam.org/family/#{rfam_id}"
       {
         order: 2,
-        title: 'Uniprot',
+        title: 'Rfam',
         url:   url,
         icon:  'fa-external-link'
       }