sequenceserver 2.0.0.rc8 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 449efaa2619fd2c83a4116480c78ae6b892804f2f9bc23c3d074b8b0493985c5
-  data.tar.gz: 393f9da8bd3df53bff3ed3a137a5d9484ad5ec9cfa1de3c7b15c3f1a6833f092
+  metadata.gz: 3cf703fae66ad96d285ed0556c387dabb0d69be120db5bdd3c3f9f8a03d3a814
+  data.tar.gz: 503bb0b69cbf773ab536d99f6782f4fff19f4872c49e6377d49a9f7820861c3b
 SHA512:
-  metadata.gz: de2fdcef61a8fcdbe8907935a1f0b545219698fe9d37fcad6cd40aaa50e779010b7e91a9158eb76590baed78ce3743cdc173b664ff5d0969dfaca42eb6bd621c
-  data.tar.gz: a65c243d19435021b00a360ddfa8f617fa58e5944dd55117a8865f11dec4554a86e577cdfee3013a82638dc45de697a33156da1ba0a2fedd8e2936a7d760d5d9
+  metadata.gz: 1896b2afddf6033c98747f7eba1dabbe15b270940bd19760ebef4ed75af267760d45d40215663173ccb2f964472b458179a459316b86ef72bc26aa75d62faad0
+  data.tar.gz: 4ac1d9a5a205d299af4b1467521a9bc539e3833d0a571cfa90b490b19df958fc0f7ee2e1341e3d8d5e19fccbe2ee6874884fc7fbcbfbdc6abbe1e60df3c9a76e

data/bin/sequenceserver

@@ -308,34 +308,6 @@ begin
             exit! unless set?
           end
         end
-      rescue SequenceServer::INCOMPATIBLE_BLAST_DATABASES => e
-        unless list_databases? || make_blast_databases?
-          # Print error raised.
-          puts
-          puts e
-
-          # Offer user to format the FASTA files.
-          database_dir = SequenceServer.config[:database_dir]
-          puts
-          puts <<~MSG
-            Incompatible databases can cause BLAST searches and other features of
-            SequenceServer to fail unexpectedly.
-            You can view incompatible databases and choose to reformat them below.
-            Alternatively, please remove them from databases directory.
-
-            View incompatible databases? [y/n] (Default: y).
-          MSG
-          puts
-          print '>> '
-          response = STDIN.gets.to_s.strip
-          unless response =~ /^[n]$/i
-            reformatted = SequenceServer.makeblastdb.reformat
-            exit! if reformatted.empty? && !set?
-            redo unless set?
-          else
-            exit! unless set?
-          end
-        end
       rescue SequenceServer::ENOENT,
         SequenceServer::CONFIG_FILE_ERROR,
         SequenceServer::BLAST_DATABASE_ERROR,
@@ -382,8 +354,28 @@ begin
 
       if make_blast_databases?
         if SequenceServer.makeblastdb.scan
-          SequenceServer.makeblastdb.run
-        else
+          puts
+          puts <<~MSG
+          SequenceServer has scanned your databases directory and will now offer
+          to convert FASTA files into BLAST databases. It will also offer to
+          reformat any old-format BLAST databases and those created without
+          the -parse_seqids option of makeblastdb (-parse_seqids option is
+          required for sequence retrieval to correctly work).
+
+          Note that reformatting process can be slow large BLAST databases and
+          fail if sequence identifiers are longer than 50 characters. While we
+          exepect the reformatting process to work in most other cases, things
+          can inevitably go wrong. Thus, please back up your databases before
+          reformatting and post any issues to our Google Group/GitHub so that
+          we can all learn from it.
+
+          Proceed? [y/n] (Default: y).
+          MSG
+          puts
+          print '>> '
+          response = STDIN.gets.to_s.strip
+          SequenceServer.makeblastdb.run unless response =~ /^[n]$/i
+          else
           puts "All FASTA files in #{SequenceServer.config[:database_dir]} " \
                'are formatted.'
         end

data/lib/sequenceserver/api_errors.rb

@@ -1,8 +1,30 @@
 module SequenceServer
+  Error = Class.new(StandardError)
+
+  # DatabaseUnreachableError is raised when the serialised Job object is
+  # refering to a database that is not present in the current filesystem.
+  class DatabaseUnreachableError < Error
+    attr_reader :more_info
+
+    def initialize(more_info)
+      super
+      @more_info = more_info
+    end
+
+    def title
+      'Sequence database unreachable'
+    end
+
+    def message
+      "The action you're trying to perform is not possible because \
+        the database is unreachable. This can happen if the database has \
+        been deleted or you are performing an action on an imported job."
+    end
+  end
+
   # API errors have an http status, title, message, and additional information
   # like stacktrace or information from program output.
-  class APIError < StandardError
-  end
+  APIError = Class.new(Error)
 
   # Job not found (404).
   class NotFound < APIError
@@ -41,7 +63,11 @@ module SequenceServer
     def message
       <<~MSG
         Looks like there's a problem with one of the query sequences, selected
-        databases, or advanced parameters.
+        databases, or advanced parameters. Details of the error are included
+        below. Please ask on our
+        <a href="https://github.com/wurmlab/sequenceserver/issues" target="_blank">issue tracker</a>
+        or on our <a href="https://groups.google.com/g/sequenceserver">forum</a> if you are
+        not sure what the error message means, or if the error message is just a number.
       MSG
     end
 

data/lib/sequenceserver/blast/constants.rb

@@ -2,7 +2,7 @@
 module SequenceServer
   # Define constanst used by BLAST module.
   module BLAST
-    ERROR_LINE = /Error:\s(.*)/
+    ERROR_LINE = /Error:\s(.*)/i
 
     ALGORITHMS = %w[blastn blastp blastx tblastn tblastx].freeze
 

data/lib/sequenceserver/blast/hit.rb

@@ -21,8 +21,8 @@ module SequenceServer
       # representation.
       def to_json(*args)
         # List all attributes that we want to send to the browser.
-        properties = %i[number id accession title length score identity
-                        qcovs sciname evalue hsps links]
+        properties = %i[number id accession title length total_score
+                        qcovs sciname hsps links]
         properties.inject({}) { |h, k| h[k] = send(k); h }.to_json(*args)
       end
 
@@ -71,23 +71,12 @@ module SequenceServer
       # client.
       ###
 
-      # Returns the sum of scores of all HSPs.
-      def score
+      # Returns the sum of scores of all HSPs. Displayed in the tabular summary
+      # of hits in the HTML report. Should probably be calculated in browser?
+      def total_score
         hsps.map(&:score).reduce(:+)
       end
 
-      # Returns the sum of identity of all HSPs divided by sum of length of all
-      # HSPs (expressed as percentage).
-      def identity
-        hsps.map(&:identity).reduce(:+) * 100 / hsps.map(&:length).reduce(:+)
-      end
-
-      # Returns the minimum evalue of all HSPs of the Hit. This is shown in the
-      # tabular overview of hits in the HTML report.
-      def evalue
-        hsps.first.evalue
-      end
-
       private
 
       # Returns the report object that this hit is a part of. This is used to

data/lib/sequenceserver/blast/job.rb

@@ -1,4 +1,5 @@
 require 'sequenceserver/job'
+require 'sequenceserver/zip_file_generator'
 
 module SequenceServer
   # BLAST module.
@@ -24,13 +25,18 @@ module SequenceServer
             @databases = Database[params[:databases]]
             @advanced  = params[:advanced].to_s.strip
             @options   = @advanced + defaults
+            # The following params are for analytics only
+            @num_threads = config[:num_threads]
+            @query_length = calculate_query_size
+            @databases_ncharacters_total = calculate_databases_ncharacters_total
           end
         end
       end
 
       # :nodoc:
       # Attributes used by us - should be considered private.
-      attr_reader :method, :qfile, :databases, :advanced, :options
+      attr_reader :advanced
+      attr_reader :databases, :databases_ncharacters_total, :method, :num_threads, :options, :qfile, :query_length
 
       # :nodoc:
       # Deprecated; see Report#extract_params
@@ -53,35 +59,25 @@ module SequenceServer
       # Override Job#raise! to raise specific API errors based on exitstatus
       # and using contents of stderr to provide context about the error.
       #
-      # rubocop:disable Metrics/CyclomaticComplexity
       def raise!
         # Return true exit status is 0 and stdout is not empty.
         return true if exitstatus.zero? && !File.zero?(stdout)
 
         # Handle error. See [1].
         case exitstatus
-        when 1
-          # Error in query sequence or options.
+        when 1..2
+          # 1: Error in query sequences or options.
+          # 2: Error in BLAST databases.
           error = IO.foreach(stderr).grep(ERROR_LINE).join
           error = File.read(stderr) if error.empty?
-          fail InputError, error
-        when 2
-          fail InputError, <<~MSG
-            BLAST signalled a problem with the databases that you searched.
-
-            Most likely one or more of your databases were created using an
-            older version of BLAST. Please consider recreating the databases
-            using BLAST #{BLAST_VERSION}.
-
-            As a temporary solution, you can try searching one database at a time.
-          MSG
+          fail InputError, "(#{exitstatus}) #{error}"
         when 4
           # Out of memory. User can retry with a shorter search, so raising
           # InputError here instead of SystemError.
           fail InputError, <<~MSG
-            Ran out of memory. Please try a smaller query, or searching fewer and smaller
+            Ran out of memory. Please try a smaller query, fewer and smaller
             databases, or limiting the output by using advanced options.
-            MSG
+          MSG
         when 6
           # Error creating output files. It can't be a permission issue as that
           # would have been caught while creating job directory. But we can run
@@ -98,13 +94,41 @@ module SequenceServer
           fail SystemError, <<~MSG
             BLAST failed abruptly (exit status: #{exitstatus}). Most likely there is a
             problem with the BLAST+ binaries.
-            MSG
+          MSG
+        end
+      end
+
+      # Use it with a block to get a self-cleaning temporary archive file
+      # of the contents of the job directory.
+      # job.as_archived_file do |tmp_file|
+      #    # do things with tmp_file
+      # end
+      def as_archived_file(&block)
+        Dir.mktmpdir(id.to_s) do |tmp_dir|
+          file_path = "#{tmp_dir}/#{id}.zip"
+          ZipFileGenerator.new(dir, file_path).write
+          File.open(file_path, 'r') do |file|
+            block.call(file)
+          end
         end
       end
-      # rubocop:enable Metrics/CyclomaticComplexity
 
       private
 
+      def calculate_databases_ncharacters_total
+        databases.map(&:ncharacters).map(&:to_i).reduce(:+)
+      end
+
+      def calculate_query_size
+        size = 0
+        IO.foreach(@qfile) do |line|
+          next if line[0] == '>'
+
+          size += line.gsub(/\s+/, '').length
+        end
+        size
+      end
+
       def validate(params)
         validate_method params[:method]
         validate_sequences params[:sequence]
@@ -118,12 +142,14 @@ module SequenceServer
 
       def validate_method(method)
         return true if ALGORITHMS.include? method
+
         fail InputError, 'BLAST algorithm should be one of:' \
                             " #{ALGORITHMS.join(', ')}."
       end
 
       def validate_sequences(sequences)
         return true if sequences.is_a?(String) && !sequences.empty?
+
         fail InputError, 'Sequences should be a non-empty string.'
       end
 
@@ -131,6 +157,7 @@ module SequenceServer
         ids = Database.ids
         return true if database_ids.is_a?(Array) && !database_ids.empty? &&
                        (ids & database_ids).length == database_ids.length
+
         fail InputError, "Database id should be one of: #{ids.join("\n")}."
       end
 
@@ -142,9 +169,7 @@ module SequenceServer
         return true if !options || (options.is_a?(String) &&
                                     options.strip.empty?)
 
-        unless allowed_chars.match(options)
-          fail InputError, 'Invalid characters detected in options.'
-        end
+        fail InputError, 'Invalid characters detected in options.' unless allowed_chars.match(options)
 
         if disallowed_options.match(options)
           failedopt = Regexp.last_match[0]
@@ -155,7 +180,7 @@ module SequenceServer
       end
 
       def allowed_chars
-        /\A[a-z0-9\-_\. ']*\Z/i
+        /\A[a-z0-9\-_. ',]*\Z/i
       end
 
       def disallowed_options
@@ -167,4 +192,4 @@ end
 
 # References
 # ----------
-# [1]: http://www.ncbi.nlm.nih.gov/books/NBK1763/
+# [1]: http://www.ncbi.nlm.nih.gov/books/NBK1763/ (Appendices)

data/lib/sequenceserver/blast/report.rb

@@ -25,6 +25,7 @@ module SequenceServer
       def initialize(job)
         super do
           @queries = []
+          @querydb = job.databases
         end
       end
 
@@ -34,15 +35,17 @@ module SequenceServer
       # Attributes parsed from job metadata and BLAST output.
       attr_reader :querydb, :dbtype, :params
 
-      def to_json
-        [:querydb, :program, :program_version, :params, :stats,
-         :queries].inject({}) { |h, k|
+      def to_json(*_args)
+        %i[querydb program program_version params stats
+           queries].inject({}) do |h, k|
           h[k] = send(k)
           h
-        }.update(search_id: job.id,
-                 submitted_at: job.submitted_at.utc,
-                 imported_xml: !!job.imported_xml_file,
-                 seqserv_version: SequenceServer::VERSION).to_json
+        end.update(search_id: job.id,
+                   submitted_at: job.submitted_at.utc,
+                   imported_xml: !job.imported_xml_file.nil?,
+                   seqserv_version: SequenceServer::VERSION,
+                   cloud_sharing_enabled: SequenceServer.config[:cloud_share_url].start_with?('http'),
+                   non_parse_seqids: !!job.databases&.any?(&:non_parse_seqids?)).to_json
       end
 
       private
@@ -54,9 +57,9 @@ module SequenceServer
         tsv_ir = nil
         if job.imported_xml_file
           xml_ir = parse_xml File.read(job.imported_xml_file)
-          tsv_ir = Hash.new do |h1,k1|
-            h1[k1] = Hash.new do |h2,k2|
-              h2[k2]=['','',[]]
+          tsv_ir = Hash.new do |h1, k1|
+            h1[k1] = Hash.new do |h2, k2|
+              h2[k2] = ['', '', []]
             end
           end
         else
@@ -80,13 +83,12 @@ module SequenceServer
       # Get database information (title and type) from job yaml or from XML.
       # Sets `querydb` and `dbtype` attributes.
       def extract_db_info(ir)
-        if job.databases.empty?
+        if @querydb.empty?
           @querydb = ir[3].split.map do |path|
             { title: File.basename(path) }
           end
           @dbtype = dbtype_from_program
         else
-          @querydb = job.databases
           @dbtype = @querydb.first.type
         end
       end
@@ -121,13 +123,13 @@ module SequenceServer
       def extract_stats(ir)
         stats  = ir[8].first[5][0]
         @stats = {
-          nsequences:   stats[0],
-          ncharacters:  stats[1],
-          hsp_length:   stats[2],
+          nsequences: stats[0],
+          ncharacters: stats[1],
+          hsp_length: stats[2],
           search_space: stats[3],
-          kappa:        stats[4],
-          labmda:       stats[5],
-          entropy:      stats[6]
+          kappa: stats[4],
+          labmda: stats[5],
+          entropy: stats[6]
         }
       end
 
@@ -143,6 +145,7 @@ module SequenceServer
       # Create Hit objects for the given query from the given ir.
       def extract_hits(xml_ir, tsv_ir, query)
         return if xml_ir == ["\n"] # => No hits.
+
         xml_ir.each do |n|
           # If hit comes from a non -parse_seqids database, then id (n[1]) is a
           # BLAST assigned internal id of the format 'gnl|BL_ORD_ID|serial'. We
@@ -168,7 +171,7 @@ module SequenceServer
       def extract_hsps(xml_ir, tsv_ir, hit)
         xml_ir.each_with_index do |n, i|
           n.insert(14, tsv_ir[i])
-          hsp = HSP.new(*[hit, *n])
+          hsp = HSP.new(hit, *n)
           hit.hsps << hsp
         end
       end
@@ -176,7 +179,9 @@ module SequenceServer
       def parse_xml(xml)
         node_to_array Ox.parse(xml).root
       rescue Ox::ParseError
-        fail InputError, <<~MSG
+        raise 'Error parsing XML file' if job.imported_xml_file
+
+        raise InputError, <<~MSG
           BLAST generated incorrect XML output. This can happen if sequence ids in your
           databases are not unique across all files. As a temporary workaround, you can
           repeat the search with one database at a time. Proper fix is to recreate the
@@ -229,9 +234,12 @@ module SequenceServer
       #    ...
       # }
       def parse_tsv(tsv)
-        ir = Hash.new {|h, k| h[k] = {} }
+        ir = Hash.new { |h, k| h[k] = {} }
         tsv.each_line do |line|
-          next if line.start_with? '#'; row = line.chomp.split("\t")
+          next if line.start_with? '#'
+
+          row = line.chomp.split("\t")
+
           (ir[row[0]][row[1]] ||= [row[2], row[3], []])[2] << row[4]
         end
         ir
@@ -244,14 +252,14 @@ module SequenceServer
 
         param_list.each_with_index do |word, i|
           nxt = param_list[i + 1]
-          if word.start_with? '-'
-            word.sub!('-', '')
-            unless nxt.nil? || nxt.start_with?('-')
-              res[word] = nxt
-            else
-              res[word] = 'True'
-            end
-          end
+          next unless word.start_with? '-'
+
+          word.sub!('-', '')
+          res[word] = unless nxt.nil? || nxt.start_with?('-')
+                        nxt
+                      else
+                        'True'
+                      end
         end
         res
       end

data/lib/sequenceserver/config.rb

@@ -11,12 +11,19 @@ module SequenceServer
 
     def initialize(data = {})
       @data = normalize data
+
       @config_file = @data.delete(:config_file)
       if @config_file
         @config_file = File.expand_path(@config_file)
-        @data = parse_config_file.update @data
+        @data = parse_config_file.deep_merge @data
       end
-      @data = defaults.update @data
+
+      @data = defaults.deep_merge @data
+
+      return unless @upgraded
+
+      logger.info 'You are using old configuration syntax. ' \
+                  'Run `sequenceserver -s` to update your config file syntax.'
     end
 
     attr_reader :data, :config_file
@@ -39,6 +46,7 @@ module SequenceServer
     # Write config data to config file.
     def write_config_file
       return unless config_file
+
       File.open(config_file, 'w') do |f|
         f.puts(data.delete_if { |_, v| v.nil? }.to_yaml)
       end
@@ -46,32 +54,34 @@ module SequenceServer
 
     private
 
-    # Symbolizes keys. Changes `database` key to `database_dir`.
+    # Symbolizes keys. Rename/reformat key-values.
     def normalize(data)
       return {} unless data
 
       # Symbolize keys.
-      data = Hash[data.map { |k, v| [k.to_sym, v] }]
+      data = data.transform_keys(&:to_sym)
 
-      # The newer config file version replaces the older database key with
-      # database_dir for correctness. Let's honour the old version as well.
+      # Very old config files may have a key called `database`.
+      # Rename it to `database_dir`
       if data[:database]
         database_dir = data.delete(:database)
         data[:database_dir] ||= database_dir
+        @upgrade = true
       end
 
-      # We would like a persistent :options: key in the config file, explicitly
-      # stating the parameters to use. Recommended values are written to config
-      # file on the first run. However, existing users won't have :options: key
-      # available from before. For them, we retain the old behaviour of
-      # automatically adding `-task blastn` for BLASTN searches.
-      if blast_opts = data.dig(:options, :blastn)
-        unless blast_opts.join.match('-task')
-          # Issue a warning.
-          logger.info "BLASTN will be run using '-task blastn' option." +
-                      " You can override this through configuration file."
-          data[:options][:blastn].push '-task blastn'
-        end
+      # Old config files may have an options hash with array values. We turn the
+      # array values into a hash. The logic is simple: If the array value is the
+      # same as default, we give it the key 'default', otherwise we give it the
+      # key 'custom'
+      data[:options]&.each do |key, val|
+        next if val.is_a? Hash
+
+        data[:options][key] = if val == defaults[:options][key][:default]
+                                { default: val }
+                              else
+                                { custom: val }
+                              end
+        @upgraded = true
       end
 
       data
@@ -86,8 +96,8 @@ module SequenceServer
       end
       logger.info "Reading configuration file: #{config_file}."
       normalize YAML.load_file(config_file)
-    rescue => error
-      raise CONFIG_FILE_ERROR.new(config_file, error)
+    rescue StandardError => e
+      raise CONFIG_FILE_ERROR.new(config_file, e)
     end
 
     def file?(file)
@@ -96,17 +106,32 @@ module SequenceServer
 
     # Default configuration data.
     def defaults
-      {
+      @defaults ||= {
         host: '0.0.0.0',
         port: 4567,
+        databases_widget: 'classic',
         options: {
-          blastn:  ['-task blastn', '-evalue 1e-5'],
-          blastp:  ['-evalue 1e-5'],
-          blastx:  ['-evalue 1e-5'],
-          tblastx: ['-evalue 1e-5'],
-          tblastn: ['-evalue 1e-5']
+          blastn: {
+            default: ['-task blastn', '-evalue 1e-5']
+          },
+          blastp: {
+            default: ['-evalue 1e-5']
+          },
+          blastx: {
+            default: ['-evalue 1e-5']
+          },
+          tblastx: {
+            default: ['-evalue 1e-5']
+          },
+          tblastn: {
+            default: ['-evalue 1e-5']
+          }
         },
-        num_threads: 1
+        num_threads: 1,
+        num_jobs: 1,
+        job_lifetime: 43_200,
+        # Set cloud_share_url to 'disabled' to disable the cloud sharing feature
+        cloud_share_url: 'https://share.sequenceserver.com/api/v1/shared-job'
       }
     end
   end