sequenceserver 3.0.1 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05745f148bb0f76da55ed9093dd77152b218f5d20656294fc879802128c24bdf
-  data.tar.gz: 76944e8d19192955b7c0f44c59466da4ff7c3db45a03190d62685abb0e5f3ccf
+  metadata.gz: 274ace0d8b4683a9bcbf6b54c83e7c28e5bc1f89b0211676f91c4d3241699148
+  data.tar.gz: d61d26277f381827f764e5d7bfd1cb61476e1e5da9dbeec9f4d9ccd4a383385c
 SHA512:
-  metadata.gz: ea823aec0b9328ebeba756547b7d9f523105a6d87230ac18e27e106e86e2e44ec9a21c3a17f01b325bc12682c35b982f3606d73689d708a8611509695a973eba
-  data.tar.gz: a0702e4ee63201fa75a366d5aa0cd7a6459d3e7e034744977d18139937008d60c68baffdb0ae0888ecc7e021b0072c1c94e2a429a0bdcb995e8738b2033c2604
+  metadata.gz: e83fe0932a797dcb8cb6f353b1256dd7e387031e078b3d3a3740662c4bd805a4f19554bf84c9bd79f4252df2decc2c6b58e911864e716944f7177014ceb4b1b4
+  data.tar.gz: fb38408e8e8da2f4f7574de8e2a0c0547072fa53b6124a68a89b2771fec533c98c01986c4928ecc2007c3f32308ba6c4bd909703d97c216cdd6d38a11fff6c97

data/bin/sequenceserver

@@ -398,8 +398,8 @@ begin
       end
 
       if import?
-        xml_file = fetch_option(:import).value
-        params = {:xml => xml_file}
+        xml_file_path = fetch_option(:import).value
+        params = {:xml => xml_file_path}
         job = SequenceServer::BLAST::Job.new(params)
         puts job.id
         exit

data/lib/sequenceserver/api_errors.rb

@@ -1,5 +1,5 @@
 module SequenceServer
-  Error = Class.new(StandardError)
+  Error = Class.new(Sinatra::Error)
 
   # DatabaseUnreachableError is raised when the serialised Job object is
   # refering to a database that is not present in the current filesystem.

data/lib/sequenceserver/blast/job.rb

@@ -21,7 +21,8 @@ module SequenceServer
         else
           validate params
           super do
-            @method    = params[:method]
+            @method = params[:method]
+            @query = params[:sequence]
             @qfile     = store('query.fa', params[:sequence])
             @databases = Database[params[:databases]]
             @advanced  = params[:advanced].to_s.strip
@@ -29,6 +30,7 @@ module SequenceServer
             # The following params are for analytics only
             @num_threads = config[:num_threads]
             @query_length = calculate_query_size
+            @number_of_query_sequences = calculate_number_of_sequences
             @databases_ncharacters_total = calculate_databases_ncharacters_total
           end
         end
@@ -36,8 +38,15 @@ module SequenceServer
 
       # :nodoc:
       # Attributes used by us - should be considered private.
-      attr_reader :advanced
-      attr_reader :databases, :databases_ncharacters_total, :method, :num_threads, :options, :qfile, :query_length
+      attr_reader :advanced,
+                  :databases,
+                  :databases_ncharacters_total,
+                  :method,
+                  :num_threads,
+                  :options,
+                  :qfile,
+                  :query_length,
+                  :number_of_query_sequences
 
       # :nodoc:
       # Deprecated; see Report#extract_params
@@ -92,6 +101,14 @@ module SequenceServer
         size
       end
 
+      def calculate_number_of_sequences
+        # splitting the query by ">" starting a new line lets us determine number of sequences
+        sequences = @query.split(/\n\s*>\s*+/)
+        # Remove any empty strings from the split result
+        sequences.reject!(&:empty?)
+        sequences.length
+      end
+
       def validate(params)
         validate_method params[:method]
         validate_sequences params[:sequence]

data/lib/sequenceserver/blast/report.rb

@@ -24,20 +24,11 @@ module SequenceServer
     class Report < Report
       def initialize(job)
         super do
-          @queries = []
           @querydb = job.databases
         end
       end
 
-      # Attributes parsed out from BLAST output.
-      attr_reader :program, :program_version, :stats, :queries
-
-      # Attributes parsed from job metadata and BLAST output.
-      attr_reader :querydb, :dbtype, :params
-
       def to_json(*_args)
-        generate
-
         %i[querydb program program_version params stats
            queries].inject({}) do |h, k|
           h[k] = send(k)
@@ -53,83 +44,89 @@ module SequenceServer
       def xml_file_size
         return File.size(job.imported_xml_file) if job.imported_xml_file
 
-        generate
-
         xml_formatter.size
       end
 
-      # Generate report.
-      def generate
-        return self if @_generated
-
-        job.raise!
-        xml_ir = nil
-        tsv_ir = nil
-        if job.imported_xml_file
-          xml_ir = parse_xml File.read(job.imported_xml_file)
-          tsv_ir = Hash.new do |h1, k1|
-            h1[k1] = Hash.new do |h2, k2|
-              h2[k2] = ['', '', []]
-            end
-          end
-        else
-          xml_ir = parse_xml(xml_formatter.read_file)
-          tsv_ir = parse_tsv(tsv_formatter.read_file)
+      def done?
+        return true if job.imported_xml_file
+
+        File.exist?(xml_formatter.filepath) && File.exist?(tsv_formatter.filepath)
+      end
+
+      def program
+        @program ||= xml_ir[0]
+      end
+
+      def program_version
+        @program_version ||= xml_ir[1]
+      end
+
+      def querydb
+        @querydb ||= xml_ir[3].split.map do |path|
+          { title: File.basename(path) }
         end
-        extract_program_info xml_ir
-        extract_db_info xml_ir
-        extract_params xml_ir
-        extract_stats xml_ir
-        extract_queries xml_ir, tsv_ir
+      end
+
+      def dbtype
+        @dbtype ||= querydb&.first&.type || dbtype_from_program
+      end
 
-        @_generated = true
+      def params
+        @params ||= extract_params
+      end
 
-        self
+      def stats
+        @stats ||= extract_stats
       end
 
-      def done?
-        return true if job.imported_xml_file
+      def queries
+        @queries ||= xml_ir[8].map do |n|
+          query = Query.new(self, n[0], n[2], n[3], [])
+          query.hits = query_hits(n[4], tsv_ir[query.id], query)
 
-        File.exist?(xml_formatter.filepath) && File.exist?(tsv_formatter.filepath)
+          query
+        end
       end
 
       private
 
-      def xml_formatter
-        @xml_formatter ||= Formatter.run(job, 'xml')
+      def xml_ir
+        @xml_ir ||=
+          if job.imported_xml_file
+            parse_xml File.read(job.imported_xml_file)
+          else
+            job.raise!
+            parse_xml(xml_formatter.read_file)
+          end
       end
 
-      def tsv_formatter
-        @tsv_formatter ||= Formatter.run(job, 'custom_tsv')
+      def tsv_ir
+        @tsv_ir ||=
+          if job.imported_xml_file
+            Hash.new do |h1, k1|
+              h1[k1] = Hash.new do |h2, k2|
+                h2[k2] = ['', '', []]
+              end
+            end
+          else
+            job.raise!
+            parse_tsv(tsv_formatter.read_file)
+          end
       end
 
-      # Make program name and program name + version available via `program`
-      # and `program_version` attributes.
-      def extract_program_info(ir)
-        @program         = ir[0]
-        @program_version = ir[1]
+      def xml_formatter
+        @xml_formatter ||= Formatter.run(job, 'xml')
       end
 
-      # Get database information (title and type) from job yaml or from XML.
-      # Sets `querydb` and `dbtype` attributes.
-      def extract_db_info(ir)
-        if @querydb.empty?
-          @querydb = ir[3].split.map do |path|
-            { title: File.basename(path) }
-          end
-          @dbtype = dbtype_from_program
-        else
-          @dbtype = @querydb.first.type
-        end
+      def tsv_formatter
+        @tsv_formatter ||= Formatter.run(job, 'custom_tsv')
       end
 
-      # Make search params available via `params` attribute.
-      #
       # Search params tweak the results. Like evalue cutoff or penalty to open
       # a gap. BLAST+ doesn't list all input params in the XML output. Only
       # matrix, evalue, gapopen, gapextend, and filters are available from XML
       # output.
-      def extract_params(ir)
+      def extract_params
         # Parse/get params from the job first.
         job_params = parse_advanced(job.advanced)
         # Old jobs from beta releases may not have the advanced key but they
@@ -138,7 +135,7 @@ module SequenceServer
 
         # Parse params from BLAST XML.
         @params = Hash[
-          *ir[7].first.map { |k, v| [k.gsub('Parameters_', ''), v] }.flatten
+          *xml_ir[7].first.map { |k, v| [k.gsub('Parameters_', ''), v] }.flatten
         ]
         @params['evalue'] = @params.delete('expect')
 
@@ -146,13 +143,11 @@ module SequenceServer
         @params = job_params.merge(@params)
       end
 
-      # Make search stats available via `stats` attribute.
-      #
       # Search stats are computed metrics. Like total number of sequences or
       # effective search space.
-      def extract_stats(ir)
-        stats  = ir[8].first[5][0]
-        @stats = {
+      def extract_stats
+        stats = xml_ir[8].first[5][0]
+        {
           nsequences: stats[0],
           ncharacters: stats[1],
           hsp_length: stats[2],
@@ -163,20 +158,11 @@ module SequenceServer
         }
       end
 
-      # Create query objects for the given report from the given ir.
-      def extract_queries(xml_ir, tsv_ir)
-        xml_ir[8].each do |n|
-          query = Query.new(self, n[0], n[2], n[3], [])
-          extract_hits(n[4], tsv_ir[query.id], query)
-          queries << query
-        end
-      end
-
       # Create Hit objects for the given query from the given ir.
-      def extract_hits(xml_ir, tsv_ir, query)
-        return if xml_ir == ["\n"] # => No hits.
+      def query_hits(xml_ir, tsv_ir, query)
+        return [] if xml_ir == ["\n"] # => No hits.
 
-        xml_ir.each do |n|
+        xml_ir.map do |n|
           # If hit comes from a non -parse_seqids database, then id (n[1]) is a
           # BLAST assigned internal id of the format 'gnl|BL_ORD_ID|serial'. We
           # assign the id to accession (because we use accession for sequence
@@ -190,19 +176,21 @@ module SequenceServer
             n[1] = defline.shift
             n[2] = defline.join(' ')
           end
+
           hit = Hit.new(query, n[0], n[1], n[3], n[2], n[4],
                         tsv_ir[n[1]][0], tsv_ir[n[1]][1], [])
-          extract_hsps(n[5], tsv_ir[n[1]][2], hit)
-          query.hits << hit
+
+          hit.hsps = hsps(n[5], tsv_ir[n[1]][2], hit)
+
+          hit
         end
       end
 
-      # Create HSP objects for the given hit from the given ir.
-      def extract_hsps(xml_ir, tsv_ir, hit)
-        xml_ir.each_with_index do |n, i|
+      def hsps(xml_ir, tsv_ir, hit)
+        xml_ir.map.with_index do |n, i|
           n.insert(14, tsv_ir[i])
-          hsp = HSP.new(hit, *n)
-          hit.hsps << hsp
+
+          HSP.new(hit, *n)
         end
       end
 

data/lib/sequenceserver/blast/tasks.rb

@@ -0,0 +1,38 @@
+module SequenceServer
+  module BLAST
+    # Shells out to each blast algorithm to get the help text and then parses it to extract the tasks.
+    module Tasks
+      ALGORITHMS = %w[blastn blastp blastx tblastn tblastx].freeze
+
+      def self.to_h
+        @to_h ||= ALGORITHMS.map do |algorithm|
+          help_text = `#{algorithm} -help`
+          [algorithm, extract_tasks(help_text)]
+        end.to_h
+      end
+
+      def self.extract_tasks(help_text)
+        lines = help_text.split("\n")
+
+        # Find task help paragraph start
+        task_line_index = lines.find_index { |line| line =~ /^\W-task/ }
+        return [] unless task_line_index.to_i.positive?
+
+        lines.slice!(0...task_line_index)
+
+        # Find the end of task help paragraph
+        next_option_line_index = lines.find_index { |line| line =~ /^\W-/ && !line.include?('-task') }
+        lines.slice!(next_option_line_index..-1)
+
+        extract_tasks_from_paragraph(lines)
+      end
+
+      def self.extract_tasks_from_paragraph(paragraph_lines)
+        as_one_liner = paragraph_lines.map(&:strip).join(' ')
+        as_one_liner.split('Permissible values:').last.split('>').first.split(' ').map do |task|
+          task.strip.gsub("'", '')
+        end.reject(&:empty?)
+      end
+    end
+  end
+end

data/lib/sequenceserver/config.rb

@@ -22,7 +22,7 @@ module SequenceServer
 
       return unless @upgraded
 
-      logger.info 'You are using old configuration syntax. ' \
+      logger.warn 'You are using old configuration syntax. ' \
                   'Run `sequenceserver -s` to update your config file syntax.'
     end
 
@@ -69,20 +69,7 @@ module SequenceServer
         @upgrade = true
       end
 
-      # Old config files may have an options hash with array values. We turn the
-      # array values into a hash. The logic is simple: If the array value is the
-      # same as default, we give it the key 'default', otherwise we give it the
-      # key 'custom'
-      data[:options]&.each do |key, val|
-        next if val.is_a? Hash
-
-        data[:options][key] = if val == defaults[:options][key][:default]
-                                { default: val }
-                              else
-                                { custom: val }
-                              end
-        @upgraded = true
-      end
+      data[:options] = convert_deprecated_options(data[:options]) if data[:options]
 
       data
     end
@@ -112,19 +99,34 @@ module SequenceServer
         databases_widget: 'classic',
         options: {
           blastn: {
-            default: ['-task blastn', '-evalue 1e-5']
+            default: {
+              description: nil,
+              attributes: ['-task blastn', '-evalue 1e-5']
+            }
           },
           blastp: {
-            default: ['-evalue 1e-5']
+            default: {
+              description: nil,
+              attributes: ['-evalue 1e-5']
+            }
           },
           blastx: {
-            default: ['-evalue 1e-5']
+            default: {
+              description: nil,
+              attributes: ['-evalue 1e-5']
+            }
           },
           tblastx: {
-            default: ['-evalue 1e-5']
+            default: {
+              description: nil,
+              attributes: ['-evalue 1e-5']
+            }
           },
           tblastn: {
-            default: ['-evalue 1e-5']
+            default: {
+              description: nil,
+              attributes: ['-evalue 1e-5']
+            }
           }
         },
         num_threads: 1,
@@ -137,5 +139,37 @@ module SequenceServer
         optimistic: false # Faster, but does not perform DB compatibility checks
       }
     end
+
+    def convert_deprecated_options(options)
+      options.each do |blast_algo, algo_config|
+        if algo_config.is_a?(Array)
+          # Very old config files may have a single array with CLI args.
+          # e.g. { blastn: ['-task blastn', '-evalue 1e-5'] }
+          # Convert the array values into a single hash naming it 'default' if
+          # the values match SequenceServer defaults.
+          options[blast_algo] = if algo_config == defaults.dig(:options, blast_algo, :default, :attributes)
+                                  { default: { attributes: algo_config } }
+                                else
+                                  { custom: { attributes: algo_config } }
+                                end
+          @upgraded = true
+        elsif algo_config.is_a?(Hash)
+          # v3.0.1 and older config files contain a flatter structure
+          # with an array instead of 'description' and 'attributes' keys.
+          # e.g. { blastn: { default: ['-task blastn', '-evalue 1e-5'] }
+          algo_config.each do |config_name, config|
+            next unless config.is_a?(Array)
+
+            options[blast_algo][config_name] = {
+              description: nil,
+              attributes: config
+            }
+            @upgraded = true
+          end
+        end
+      end
+
+      options
+    end
   end
 end

data/lib/sequenceserver/makeblastdb.rb

@@ -13,6 +13,7 @@ module SequenceServer
   #
   class MAKEBLASTDB
     extend Forwardable
+    GUESS_SAMPLE_SIZE = 1_048_576
 
     def_delegators SequenceServer, :config, :sys
 
@@ -333,8 +334,21 @@ module SequenceServer
     # If the given file is FASTA, returns Array of as many different
     # sequences in the portion of the file read. Returns the portion
     # of the file read wrapped in an Array otherwise.
-    def sample_sequences(file)
-      File.read(file, 1_048_576).split(/^>.+$/).delete_if(&:empty?)
+    def sample_sequences(file, offset = 0)
+      sample = File.read(file, GUESS_SAMPLE_SIZE, offset)
+
+      return [] if sample.nil?
+
+      # remove all unknown bases (indicated by 'N') before sampling
+      sample.gsub!(/N/, '')
+      meaningful_samples = sample.split(/^>.+$/).map { |line| line.gsub(/^\n+$/, '') }.delete_if(&:empty?)
+
+      if meaningful_samples.empty?
+        offset += GUESS_SAMPLE_SIZE
+        sample_sequences(file, offset)
+      else
+        meaningful_samples
+      end
     end
   end
 end

data/lib/sequenceserver/report.rb

@@ -6,12 +6,6 @@ module SequenceServer
   # Report is a generic superclass. Programs, like BLAST, must implement their
   # own report subclass.
   class Report
-    class << self
-      def generate(job)
-        BLAST::Report.new(job).generate
-      end
-    end
-
     # Provide access to global `config` & `logger` services to the report
     # objects.
     extend Forwardable

data/lib/sequenceserver/routes.rb

@@ -5,6 +5,7 @@ require 'rest-client'
 
 require 'sequenceserver/job'
 require 'sequenceserver/blast'
+require 'sequenceserver/blast/tasks'
 require 'sequenceserver/report'
 require 'sequenceserver/database'
 require 'sequenceserver/sequence'
@@ -32,6 +33,8 @@ module SequenceServer
       # Override in config.ru if the instance is served under a subpath
       # e.g. for example.org/our-sequenceserver set to '/our-sequenceserver'
       set :root_path_prefix, ''
+
+      set :search_layout, :'search_layout'
     end
 
     # See
@@ -74,7 +77,7 @@ module SequenceServer
 
     # Returns base HTML. Rest happens client-side: rendering the search form.
     get '/' do
-      erb :search, layout: true
+      erb :search, layout: settings.search_layout
     end
 
     # Returns data that is used to render the search form client side. These
@@ -83,7 +86,8 @@ module SequenceServer
       searchdata = {
         query: Database.retrieve(params[:query]),
         database: Database.all,
-        options: SequenceServer.config[:options]
+        options: SequenceServer.config[:options],
+        blastTaskMap: SequenceServer::BLAST::Tasks.to_h
       }
 
       searchdata.update(tree: Database.tree) if SequenceServer.config[:databases_widget] == 'tree'
@@ -100,7 +104,7 @@ module SequenceServer
     post '/' do
       if params[:input_sequence]
         @input_sequence = params[:input_sequence]
-        erb :search, layout: true
+        erb :search, layout: settings.search_layout
       else
         job = Job.create(params)
         redirect to("/#{job.id}")
@@ -114,24 +118,11 @@ module SequenceServer
       halt 404, { error: 'Job not found' }.to_json if job.nil?
       halt 202 unless job.done?
 
-      report = Report.generate(job)
+      report = BLAST::Report.new(job)
       halt 202 unless report.done?
 
-      display_large_result_warning =
-        SequenceServer.config[:large_result_warning_threshold].to_i.positive? &&
-        params[:bypass_file_size_warning] != 'true' &&
-        report.xml_file_size > SequenceServer.config[:large_result_warning_threshold]
-
-      if display_large_result_warning
-        halt 200,
-             {
-               user_warning: 'LARGE_RESULT',
-               download_links: [
-                 { name: 'Standard Tabular Report', url: "download/#{jid}.std_tsv" },
-                 { name: 'Full Tabular Report', url: "/download/#{jid}.full_tsv" },
-                 { name: 'Results in XML', url: "/download/#{jid}.xml" }
-               ]
-             }.to_json
+      if display_large_result_warning?(report.xml_file_size)
+        halt 200, large_result_warning_payload(jid).to_json
       end
 
       report.to_json
@@ -307,12 +298,32 @@ module SequenceServer
       # the user hits the back button. Thus we do not test for empty string.
       method = job.method.to_sym
       if job.advanced && job.advanced !=
-                         searchdata[:options][method][:default].join(' ')
+                         searchdata.dig(:options, method, :default, :attributes).to_a.join(' ')
         searchdata[:options] = searchdata[:options].deep_copy
-        searchdata[:options][method]['last search'] = [job.advanced]
+        searchdata[:options][method]['last search'] = { attributes: [job.advanced] }
       end
     end
 
+    def display_large_result_warning?(xml_file_size)
+      threshold = SequenceServer.config[:large_result_warning_threshold].to_i
+      return false unless threshold.positive?
+
+      return false if params[:bypass_file_size_warning] == 'true'
+
+      xml_file_size > threshold
+    end
+
+    def large_result_warning_payload(jid)
+      {
+        user_warning: 'LARGE_RESULT',
+        download_links: [
+          { name: 'Standard Tabular Report', url: "download/#{jid}.std_tsv" },
+          { name: 'Full Tabular Report', url: "/download/#{jid}.full_tsv" },
+          { name: 'Results in XML', url: "/download/#{jid}.xml" }
+        ]
+      }
+    end
+
     helpers do
       def root_path_prefix
         settings.root_path_prefix.to_s

data/lib/sequenceserver/version.rb

@@ -1,4 +1,4 @@
 # Define version number.
 module SequenceServer
-  VERSION = '3.0.1'.freeze
+  VERSION = '3.1.0'.freeze
 end

data/lib/sequenceserver.rb

@@ -8,7 +8,7 @@ Encoding.default_internal = 'UTF-8'
 # Top level module / namespace.
 module SequenceServer
   # The default version of BLAST that will be downloaded and configured for use.
-  BLAST_VERSION = '2.12.0+'.freeze
+  BLAST_VERSION = '2.15.0+'.freeze
 
   # Default location of configuration file.
   DEFAULT_CONFIG_FILE = '~/.sequenceserver.conf'.freeze