sequenceserver 3.0.1 → 3.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05745f148bb0f76da55ed9093dd77152b218f5d20656294fc879802128c24bdf
-  data.tar.gz: 76944e8d19192955b7c0f44c59466da4ff7c3db45a03190d62685abb0e5f3ccf
+  metadata.gz: 1579fdcd4e8dc694025ccde8534ab3a48d935cfef4830a8de2c446657449aaf8
+  data.tar.gz: 25561e738f1511d4ee40730eec04c943148749313d29ccf362098b15a1e922c5
 SHA512:
-  metadata.gz: ea823aec0b9328ebeba756547b7d9f523105a6d87230ac18e27e106e86e2e44ec9a21c3a17f01b325bc12682c35b982f3606d73689d708a8611509695a973eba
-  data.tar.gz: a0702e4ee63201fa75a366d5aa0cd7a6459d3e7e034744977d18139937008d60c68baffdb0ae0888ecc7e021b0072c1c94e2a429a0bdcb995e8738b2033c2604
+  metadata.gz: 965a7ab34577a0180d8ac1ac830c84737118e4c3fe2997cc61886424929066a95d3df3869c01ca46ffceb3a9ef3d39b96676e87841bf57c0c4903cd99700fde0
+  data.tar.gz: 3a5fa2dce619076a5e0b626981c2bb95ffc544ae8b920942b2979cabe4869120e549f494cbd15a02bf10e265bb303f8988264824839ea934f80797e280b8045a

data/bin/sequenceserver

@@ -398,8 +398,8 @@ begin
       end
 
       if import?
-        xml_file = fetch_option(:import).value
-        params = {:xml => xml_file}
+        xml_file_path = fetch_option(:import).value
+        params = {:xml => xml_file_path}
         job = SequenceServer::BLAST::Job.new(params)
         puts job.id
         exit

data/lib/sequenceserver/api_errors.rb

@@ -1,9 +1,10 @@
 module SequenceServer
-  Error = Class.new(StandardError)
+  Error = Class.new(Sinatra::Error)
 
+  ValidationError = Class.new(Error)
   # DatabaseUnreachableError is raised when the serialised Job object is
   # refering to a database that is not present in the current filesystem.
-  class DatabaseUnreachableError < Error
+  class DatabaseUnreachableError < ValidationError
     attr_reader :more_info
 
     def initialize(more_info)
@@ -11,6 +12,10 @@ module SequenceServer
       @more_info = more_info
     end
 
+    def http_status
+      422
+    end
+
     def title
       'Sequence database unreachable'
     end
@@ -22,6 +27,55 @@ module SequenceServer
     end
   end
 
+  # InvalidSequenceIdError is raised when the FASTA sequence ID provided by the
+  # frontend appears to be invalid. It is important to validate the sequence ID
+  # format for security reasons.
+  class InvalidSequenceIdError < ValidationError
+    attr_reader :more_info
+
+    def initialize(more_info)
+      super
+      @more_info = more_info
+    end
+
+    def http_status
+      422
+    end
+
+    def title
+      'Sequence ID invalid'
+    end
+
+    def message
+      "The action you're trying to perform is not possible because \
+        one of the FASTA ids seems to be invalid."
+    end
+  end
+
+  # InvalidParameterError is a more generic error class that can be
+  # raised when the frontend sends a request with an invalid parameter
+  class InvalidParameterError < ValidationError
+    attr_reader :more_info
+
+    def initialize(more_info)
+      super
+      @more_info = more_info
+    end
+
+    def http_status
+      422
+    end
+
+    def title
+      'Invalid parameter'
+    end
+
+    def message
+      "The action you're trying to perform is not possible because \
+        one of the provided parameters is invalid."
+    end
+  end
+
   # API errors have an http status, title, message, and additional information
   # like stacktrace or information from program output.
   APIError = Class.new(Error)

data/lib/sequenceserver/blast/job.rb

@@ -21,7 +21,8 @@ module SequenceServer
         else
           validate params
           super do
-            @method    = params[:method]
+            @method = params[:method]
+            @query = params[:sequence]
             @qfile     = store('query.fa', params[:sequence])
             @databases = Database[params[:databases]]
             @advanced  = params[:advanced].to_s.strip
@@ -29,6 +30,7 @@ module SequenceServer
             # The following params are for analytics only
             @num_threads = config[:num_threads]
             @query_length = calculate_query_size
+            @number_of_query_sequences = calculate_number_of_sequences
             @databases_ncharacters_total = calculate_databases_ncharacters_total
           end
         end
@@ -36,8 +38,15 @@ module SequenceServer
 
       # :nodoc:
       # Attributes used by us - should be considered private.
-      attr_reader :advanced
-      attr_reader :databases, :databases_ncharacters_total, :method, :num_threads, :options, :qfile, :query_length
+      attr_reader :advanced,
+                  :databases,
+                  :databases_ncharacters_total,
+                  :method,
+                  :num_threads,
+                  :options,
+                  :qfile,
+                  :query_length,
+                  :number_of_query_sequences
 
       # :nodoc:
       # Deprecated; see Report#extract_params
@@ -92,6 +101,14 @@ module SequenceServer
         size
       end
 
+      def calculate_number_of_sequences
+        # splitting the query by ">" starting a new line lets us determine number of sequences
+        sequences = @query.split(/\n\s*>\s*+/)
+        # Remove any empty strings from the split result
+        sequences.reject!(&:empty?)
+        sequences.length
+      end
+
       def validate(params)
         validate_method params[:method]
         validate_sequences params[:sequence]

data/lib/sequenceserver/blast/report.rb

@@ -24,20 +24,11 @@ module SequenceServer
     class Report < Report
       def initialize(job)
         super do
-          @queries = []
           @querydb = job.databases
         end
       end
 
-      # Attributes parsed out from BLAST output.
-      attr_reader :program, :program_version, :stats, :queries
-
-      # Attributes parsed from job metadata and BLAST output.
-      attr_reader :querydb, :dbtype, :params
-
       def to_json(*_args)
-        generate
-
         %i[querydb program program_version params stats
            queries].inject({}) do |h, k|
           h[k] = send(k)
@@ -53,83 +44,89 @@ module SequenceServer
       def xml_file_size
         return File.size(job.imported_xml_file) if job.imported_xml_file
 
-        generate
-
         xml_formatter.size
       end
 
-      # Generate report.
-      def generate
-        return self if @_generated
-
-        job.raise!
-        xml_ir = nil
-        tsv_ir = nil
-        if job.imported_xml_file
-          xml_ir = parse_xml File.read(job.imported_xml_file)
-          tsv_ir = Hash.new do |h1, k1|
-            h1[k1] = Hash.new do |h2, k2|
-              h2[k2] = ['', '', []]
-            end
-          end
-        else
-          xml_ir = parse_xml(xml_formatter.read_file)
-          tsv_ir = parse_tsv(tsv_formatter.read_file)
+      def done?
+        return true if job.imported_xml_file
+
+        File.exist?(xml_formatter.filepath) && File.exist?(tsv_formatter.filepath)
+      end
+
+      def program
+        @program ||= xml_ir[0]
+      end
+
+      def program_version
+        @program_version ||= xml_ir[1]
+      end
+
+      def querydb
+        @querydb ||= xml_ir[3].split.map do |path|
+          { title: File.basename(path) }
         end
-        extract_program_info xml_ir
-        extract_db_info xml_ir
-        extract_params xml_ir
-        extract_stats xml_ir
-        extract_queries xml_ir, tsv_ir
+      end
+
+      def dbtype
+        @dbtype ||= querydb&.first&.type || dbtype_from_program
+      end
 
-        @_generated = true
+      def params
+        @params ||= extract_params
+      end
 
-        self
+      def stats
+        @stats ||= extract_stats
       end
 
-      def done?
-        return true if job.imported_xml_file
+      def queries
+        @queries ||= xml_ir[8].map do |n|
+          query = Query.new(self, n[0], n[2], n[3], [])
+          query.hits = query_hits(n[4], tsv_ir[query.id], query)
 
-        File.exist?(xml_formatter.filepath) && File.exist?(tsv_formatter.filepath)
+          query
+        end
       end
 
       private
 
-      def xml_formatter
-        @xml_formatter ||= Formatter.run(job, 'xml')
+      def xml_ir
+        @xml_ir ||=
+          if job.imported_xml_file
+            parse_xml File.read(job.imported_xml_file)
+          else
+            job.raise!
+            parse_xml(xml_formatter.read_file)
+          end
       end
 
-      def tsv_formatter
-        @tsv_formatter ||= Formatter.run(job, 'custom_tsv')
+      def tsv_ir
+        @tsv_ir ||=
+          if job.imported_xml_file
+            Hash.new do |h1, k1|
+              h1[k1] = Hash.new do |h2, k2|
+                h2[k2] = ['', '', []]
+              end
+            end
+          else
+            job.raise!
+            parse_tsv(tsv_formatter.read_file)
+          end
       end
 
-      # Make program name and program name + version available via `program`
-      # and `program_version` attributes.
-      def extract_program_info(ir)
-        @program         = ir[0]
-        @program_version = ir[1]
+      def xml_formatter
+        @xml_formatter ||= Formatter.run(job, 'xml')
       end
 
-      # Get database information (title and type) from job yaml or from XML.
-      # Sets `querydb` and `dbtype` attributes.
-      def extract_db_info(ir)
-        if @querydb.empty?
-          @querydb = ir[3].split.map do |path|
-            { title: File.basename(path) }
-          end
-          @dbtype = dbtype_from_program
-        else
-          @dbtype = @querydb.first.type
-        end
+      def tsv_formatter
+        @tsv_formatter ||= Formatter.run(job, 'custom_tsv')
       end
 
-      # Make search params available via `params` attribute.
-      #
       # Search params tweak the results. Like evalue cutoff or penalty to open
       # a gap. BLAST+ doesn't list all input params in the XML output. Only
       # matrix, evalue, gapopen, gapextend, and filters are available from XML
       # output.
-      def extract_params(ir)
+      def extract_params
         # Parse/get params from the job first.
         job_params = parse_advanced(job.advanced)
         # Old jobs from beta releases may not have the advanced key but they
@@ -138,7 +135,7 @@ module SequenceServer
 
         # Parse params from BLAST XML.
         @params = Hash[
-          *ir[7].first.map { |k, v| [k.gsub('Parameters_', ''), v] }.flatten
+          *xml_ir[7].first.map { |k, v| [k.gsub('Parameters_', ''), v] }.flatten
         ]
         @params['evalue'] = @params.delete('expect')
 
@@ -146,13 +143,11 @@ module SequenceServer
         @params = job_params.merge(@params)
       end
 
-      # Make search stats available via `stats` attribute.
-      #
       # Search stats are computed metrics. Like total number of sequences or
       # effective search space.
-      def extract_stats(ir)
-        stats  = ir[8].first[5][0]
-        @stats = {
+      def extract_stats
+        stats = xml_ir[8].first[5][0]
+        {
           nsequences: stats[0],
           ncharacters: stats[1],
           hsp_length: stats[2],
@@ -163,20 +158,11 @@ module SequenceServer
         }
       end
 
-      # Create query objects for the given report from the given ir.
-      def extract_queries(xml_ir, tsv_ir)
-        xml_ir[8].each do |n|
-          query = Query.new(self, n[0], n[2], n[3], [])
-          extract_hits(n[4], tsv_ir[query.id], query)
-          queries << query
-        end
-      end
-
       # Create Hit objects for the given query from the given ir.
-      def extract_hits(xml_ir, tsv_ir, query)
-        return if xml_ir == ["\n"] # => No hits.
+      def query_hits(xml_ir, tsv_ir, query)
+        return [] if xml_ir == ["\n"] # => No hits.
 
-        xml_ir.each do |n|
+        xml_ir.map do |n|
           # If hit comes from a non -parse_seqids database, then id (n[1]) is a
           # BLAST assigned internal id of the format 'gnl|BL_ORD_ID|serial'. We
           # assign the id to accession (because we use accession for sequence
@@ -190,19 +176,21 @@ module SequenceServer
             n[1] = defline.shift
             n[2] = defline.join(' ')
           end
+
           hit = Hit.new(query, n[0], n[1], n[3], n[2], n[4],
                         tsv_ir[n[1]][0], tsv_ir[n[1]][1], [])
-          extract_hsps(n[5], tsv_ir[n[1]][2], hit)
-          query.hits << hit
+
+          hit.hsps = hsps(n[5], tsv_ir[n[1]][2], hit)
+
+          hit
         end
       end
 
-      # Create HSP objects for the given hit from the given ir.
-      def extract_hsps(xml_ir, tsv_ir, hit)
-        xml_ir.each_with_index do |n, i|
+      def hsps(xml_ir, tsv_ir, hit)
+        xml_ir.map.with_index do |n, i|
           n.insert(14, tsv_ir[i])
-          hsp = HSP.new(hit, *n)
-          hit.hsps << hsp
+
+          HSP.new(hit, *n)
         end
       end
 

data/lib/sequenceserver/blast/tasks.rb

@@ -0,0 +1,38 @@
+module SequenceServer
+  module BLAST
+    # Shells out to each blast algorithm to get the help text and then parses it to extract the tasks.
+    module Tasks
+      ALGORITHMS = %w[blastn blastp blastx tblastn tblastx].freeze
+
+      def self.to_h
+        @to_h ||= ALGORITHMS.map do |algorithm|
+          help_text = `#{algorithm} -help`
+          [algorithm, extract_tasks(help_text)]
+        end.to_h
+      end
+
+      def self.extract_tasks(help_text)
+        lines = help_text.split("\n")
+
+        # Find task help paragraph start
+        task_line_index = lines.find_index { |line| line =~ /^\W-task/ }
+        return [] unless task_line_index.to_i.positive?
+
+        lines.slice!(0...task_line_index)
+
+        # Find the end of task help paragraph
+        next_option_line_index = lines.find_index { |line| line =~ /^\W-/ && !line.include?('-task') }
+        lines.slice!(next_option_line_index..-1)
+
+        extract_tasks_from_paragraph(lines)
+      end
+
+      def self.extract_tasks_from_paragraph(paragraph_lines)
+        as_one_liner = paragraph_lines.map(&:strip).join(' ')
+        as_one_liner.split('Permissible values:').last.split('>').first.split(' ').map do |task|
+          task.strip.gsub("'", '')
+        end.reject(&:empty?)
+      end
+    end
+  end
+end

data/lib/sequenceserver/blast.rb

@@ -1,3 +1,9 @@
 require_relative 'blast/job'
 require_relative 'blast/report'
 require_relative 'blast/constants'
+
+module SequenceServer
+  module BLAST
+    VALID_SEQUENCE_ID = /\A[a-zA-Z0-9\-_.:*#|\[\]]+\z/
+  end
+end

data/lib/sequenceserver/config.rb

@@ -22,7 +22,7 @@ module SequenceServer
 
       return unless @upgraded
 
-      logger.info 'You are using old configuration syntax. ' \
+      logger.warn 'You are using old configuration syntax. ' \
                   'Run `sequenceserver -s` to update your config file syntax.'
     end
 
@@ -69,20 +69,7 @@ module SequenceServer
         @upgrade = true
       end
 
-      # Old config files may have an options hash with array values. We turn the
-      # array values into a hash. The logic is simple: If the array value is the
-      # same as default, we give it the key 'default', otherwise we give it the
-      # key 'custom'
-      data[:options]&.each do |key, val|
-        next if val.is_a? Hash
-
-        data[:options][key] = if val == defaults[:options][key][:default]
-                                { default: val }
-                              else
-                                { custom: val }
-                              end
-        @upgraded = true
-      end
+      data[:options] = convert_deprecated_options(data[:options]) if data[:options]
 
       data
     end
@@ -112,19 +99,34 @@ module SequenceServer
         databases_widget: 'classic',
         options: {
           blastn: {
-            default: ['-task blastn', '-evalue 1e-5']
+            default: {
+              description: nil,
+              attributes: ['-task blastn', '-evalue 1e-5']
+            }
           },
           blastp: {
-            default: ['-evalue 1e-5']
+            default: {
+              description: nil,
+              attributes: ['-evalue 1e-5']
+            }
           },
           blastx: {
-            default: ['-evalue 1e-5']
+            default: {
+              description: nil,
+              attributes: ['-evalue 1e-5']
+            }
           },
           tblastx: {
-            default: ['-evalue 1e-5']
+            default: {
+              description: nil,
+              attributes: ['-evalue 1e-5']
+            }
           },
           tblastn: {
-            default: ['-evalue 1e-5']
+            default: {
+              description: nil,
+              attributes: ['-evalue 1e-5']
+            }
           }
         },
         num_threads: 1,
@@ -137,5 +139,37 @@ module SequenceServer
         optimistic: false # Faster, but does not perform DB compatibility checks
       }
     end
+
+    def convert_deprecated_options(options)
+      options.each do |blast_algo, algo_config|
+        if algo_config.is_a?(Array)
+          # Very old config files may have a single array with CLI args.
+          # e.g. { blastn: ['-task blastn', '-evalue 1e-5'] }
+          # Convert the array values into a single hash naming it 'default' if
+          # the values match SequenceServer defaults.
+          options[blast_algo] = if algo_config == defaults.dig(:options, blast_algo, :default, :attributes)
+                                  { default: { attributes: algo_config } }
+                                else
+                                  { custom: { attributes: algo_config } }
+                                end
+          @upgraded = true
+        elsif algo_config.is_a?(Hash)
+          # v3.0.1 and older config files contain a flatter structure
+          # with an array instead of 'description' and 'attributes' keys.
+          # e.g. { blastn: { default: ['-task blastn', '-evalue 1e-5'] }
+          algo_config.each do |config_name, config|
+            next unless config.is_a?(Array)
+
+            options[blast_algo][config_name] = {
+              description: nil,
+              attributes: config
+            }
+            @upgraded = true
+          end
+        end
+      end
+
+      options
+    end
   end
 end

data/lib/sequenceserver/database.rb

@@ -37,8 +37,19 @@ module SequenceServer
     alias path name
 
     def retrieve(accession, coords = nil)
+      fail(
+        InvalidSequenceIdError,
+        "Invalid sequence id: #{accession}"
+      ) unless accession =~ SequenceServer::BLAST::VALID_SEQUENCE_ID
+
       cmd = "blastdbcmd -db #{name} -entry '#{accession}'"
+
       if coords
+        fail(
+          InvalidParameterError,
+          "Invalid range coordinates: #{coords}"
+        ) unless coords =~ /[0-9]+-[0-9]*/
+
         cmd << " -range #{coords}"
       end
       out, = sys(cmd, path: config[:bin])
@@ -52,6 +63,8 @@ module SequenceServer
     # Returns true if the database contains the given sequence id.
     # Returns false otherwise.
     def include?(id)
+      fail ArgumentError, "Invalid sequence id: #{id}" unless id =~ SequenceServer::BLAST::VALID_SEQUENCE_ID
+
       cmd = "blastdbcmd -entry '#{id}' -db #{name}"
       sys(cmd, path: config[:bin]) rescue false
     end

data/lib/sequenceserver/makeblastdb.rb

@@ -13,6 +13,7 @@ module SequenceServer
   #
   class MAKEBLASTDB
     extend Forwardable
+    GUESS_SAMPLE_SIZE = 1_048_576
 
     def_delegators SequenceServer, :config, :sys
 
@@ -333,8 +334,21 @@ module SequenceServer
     # If the given file is FASTA, returns Array of as many different
     # sequences in the portion of the file read. Returns the portion
     # of the file read wrapped in an Array otherwise.
-    def sample_sequences(file)
-      File.read(file, 1_048_576).split(/^>.+$/).delete_if(&:empty?)
+    def sample_sequences(file, offset = 0)
+      sample = File.read(file, GUESS_SAMPLE_SIZE, offset)
+
+      return [] if sample.nil?
+
+      # remove all unknown bases (indicated by 'N') before sampling
+      sample.gsub!(/N/, '')
+      meaningful_samples = sample.split(/^>.+$/).map { |line| line.gsub(/^\n+$/, '') }.delete_if(&:empty?)
+
+      if meaningful_samples.empty?
+        offset += GUESS_SAMPLE_SIZE
+        sample_sequences(file, offset)
+      else
+        meaningful_samples
+      end
     end
   end
 end

data/lib/sequenceserver/report.rb

@@ -6,12 +6,6 @@ module SequenceServer
   # Report is a generic superclass. Programs, like BLAST, must implement their
   # own report subclass.
   class Report
-    class << self
-      def generate(job)
-        BLAST::Report.new(job).generate
-      end
-    end
-
     # Provide access to global `config` & `logger` services to the report
     # objects.
     extend Forwardable