sequenceserver 2.0.0 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25c2424857780517682276d21694f8ab0c3594772b82f08dcdca1934dcf23fd9
-  data.tar.gz: 3e27329d829825b7a3a73f14ebdecad514648381dbba914cfca6ee63564c795a
+  metadata.gz: 61b15f95cdc065739de70f2ba94d3022e6beec789dd9c64199ee1be3a2459c63
+  data.tar.gz: d6c108bd39cb8c6a832787d2cbaa52b2c5a3653c0b08b289cb149b7221a9870c
 SHA512:
-  metadata.gz: 5137bf6118ee0044a0b8d5b1979f525686d7dd1698d6c795682d39a6c9d8247410104441fb642a585e6df9a18ad827336c8f61215f9e3c8c68fb387e0b6e2f41
-  data.tar.gz: e7ec5cebd2595b3a5f90cb750ce583161821e1be22dfcf14b1ba1fbe11dda440105795ac04b06caeb456c0f78e55db50d730704db8972667ca8d7acae5170692
+  metadata.gz: d4c2b1785980dd4e46a24a8a5a6d36652fb23f5389f9b7320cd9360f081cbad82b6dc6bfe735b5321738611094052631af0de635b77676587d703d466bd5f604
+  data.tar.gz: 235e35e8ba4331894a7ac9f107d950462d3b3d892ddafc6f58d3ba48fbe6a03cfc016ac64798921472e296a0cb5577245aca49cbb517d897094a501ec4483beb

data/bin/sequenceserver

@@ -119,6 +119,11 @@ begin
        'Port to run SequenceServer on',
        argument: true
 
+    on 'o', 'optimistic=',
+       'Optimistic mode does not perform database compatibility checks, which can be faster. ' \
+       'Only use this if you format FASTAs with "sequenceserver -make-blast-databases"',
+       argument: true
+
     on 's', 'set',
        'Set configuration value in default or given config file'
 
@@ -296,7 +301,7 @@ begin
           unless response =~ /^[n]$/i
             puts
             puts 'Searching ...'
-            if SequenceServer.makeblastdb.any_unformatted?
+            if SequenceServer.makeblastdb.any_to_format?
               formatted = SequenceServer.makeblastdb.format
               exit! if formatted.empty? && !set?
               redo unless set?
@@ -353,7 +358,7 @@ begin
       end
 
       if make_blast_databases?
-        if SequenceServer.makeblastdb.scan
+        if SequenceServer.makeblastdb.any_to_format_or_reformat?
           puts
           puts <<~MSG
           SequenceServer has scanned your databases directory and will now offer

data/lib/sequenceserver/api_errors.rb

@@ -1,8 +1,30 @@
 module SequenceServer
+  Error = Class.new(StandardError)
+
+  # DatabaseUnreachableError is raised when the serialised Job object is
+  # refering to a database that is not present in the current filesystem.
+  class DatabaseUnreachableError < Error
+    attr_reader :more_info
+
+    def initialize(more_info)
+      super
+      @more_info = more_info
+    end
+
+    def title
+      'Sequence database unreachable'
+    end
+
+    def message
+      "The action you're trying to perform is not possible because \
+        the database is unreachable. This can happen if the database has \
+        been deleted or you are performing an action on an imported job."
+    end
+  end
+
   # API errors have an http status, title, message, and additional information
   # like stacktrace or information from program output.
-  class APIError < StandardError
-  end
+  APIError = Class.new(Error)
 
   # Job not found (404).
   class NotFound < APIError

data/lib/sequenceserver/blast/formatter.rb

@@ -25,23 +25,32 @@ module SequenceServer
 
       attr_reader :format, :mime, :specifiers
 
-      def file
-        @file ||= File.join(job.dir, filename)
+      def filepath
+        @filepath ||= File.join(job.dir, filename)
+      end
+
+      def size
+        File.size(filepath)
       end
 
       def filename
         @filename ||= "sequenceserver-#{type}_report.#{mime}"
       end
 
+      def read_file
+        File.read(filepath)
+      end
+
       private
 
       attr_reader :job, :type
 
       def run
-        return if File.exist?(file)
+        return if File.exist?(filepath)
+
         command = "blast_formatter -archive '#{job.stdout}'" \
           " -outfmt '#{format} #{specifiers}'"
-        sys(command, path: config[:bin], dir: DOTDIR, stdout: file)
+        sys(command, path: config[:bin], dir: DOTDIR, stdout: filepath)
       rescue CommandFailed => e
         # Mostly we will never get here: empty archive file,
         # file permissions, broken BLAST binaries, etc. will

data/lib/sequenceserver/blast/job.rb

@@ -1,4 +1,5 @@
 require 'sequenceserver/job'
+require 'sequenceserver/zip_file_generator'
 
 module SequenceServer
   # BLAST module.
@@ -24,13 +25,18 @@ module SequenceServer
             @databases = Database[params[:databases]]
             @advanced  = params[:advanced].to_s.strip
             @options   = @advanced + defaults
+            # The following params are for analytics only
+            @num_threads = config[:num_threads]
+            @query_length = calculate_query_size
+            @databases_ncharacters_total = calculate_databases_ncharacters_total
           end
         end
       end
 
       # :nodoc:
       # Attributes used by us - should be considered private.
-      attr_reader :method, :qfile, :databases, :advanced, :options
+      attr_reader :advanced
+      attr_reader :databases, :databases_ncharacters_total, :method, :num_threads, :options, :qfile, :query_length
 
       # :nodoc:
       # Deprecated; see Report#extract_params
@@ -53,7 +59,6 @@ module SequenceServer
       # Override Job#raise! to raise specific API errors based on exitstatus
       # and using contents of stderr to provide context about the error.
       #
-      # rubocop:disable Metrics/CyclomaticComplexity
       def raise!
         # Return true exit status is 0 and stdout is not empty.
         return true if exitstatus.zero? && !File.zero?(stdout)
@@ -92,10 +97,38 @@ module SequenceServer
           MSG
         end
       end
-      # rubocop:enable Metrics/CyclomaticComplexity
+
+      # Use it with a block to get a self-cleaning temporary archive file
+      # of the contents of the job directory.
+      # job.as_archived_file do |tmp_file|
+      #    # do things with tmp_file
+      # end
+      def as_archived_file(&block)
+        Dir.mktmpdir(id.to_s) do |tmp_dir|
+          file_path = "#{tmp_dir}/#{id}.zip"
+          ZipFileGenerator.new(dir, file_path).write
+          File.open(file_path, 'r') do |file|
+            block.call(file)
+          end
+        end
+      end
 
       private
 
+      def calculate_databases_ncharacters_total
+        databases.map(&:ncharacters).map(&:to_i).reduce(:+)
+      end
+
+      def calculate_query_size
+        size = 0
+        IO.foreach(@qfile) do |line|
+          next if line[0] == '>'
+
+          size += line.gsub(/\s+/, '').length
+        end
+        size
+      end
+
       def validate(params)
         validate_method params[:method]
         validate_sequences params[:sequence]
@@ -109,12 +142,14 @@ module SequenceServer
 
       def validate_method(method)
         return true if ALGORITHMS.include? method
+
         fail InputError, 'BLAST algorithm should be one of:' \
                             " #{ALGORITHMS.join(', ')}."
       end
 
       def validate_sequences(sequences)
         return true if sequences.is_a?(String) && !sequences.empty?
+
         fail InputError, 'Sequences should be a non-empty string.'
       end
 
@@ -122,6 +157,7 @@ module SequenceServer
         ids = Database.ids
         return true if database_ids.is_a?(Array) && !database_ids.empty? &&
                        (ids & database_ids).length == database_ids.length
+
         fail InputError, "Database id should be one of: #{ids.join("\n")}."
       end
 
@@ -133,9 +169,7 @@ module SequenceServer
         return true if !options || (options.is_a?(String) &&
                                     options.strip.empty?)
 
-        unless allowed_chars.match(options)
-          fail InputError, 'Invalid characters detected in options.'
-        end
+        fail InputError, 'Invalid characters detected in options.' unless allowed_chars.match(options)
 
         if disallowed_options.match(options)
           failedopt = Regexp.last_match[0]
@@ -146,7 +180,7 @@ module SequenceServer
       end
 
       def allowed_chars
-        /\A[a-z0-9\-_\. ',]*\Z/i
+        /\A[a-z0-9\-_. ',]*\Z/i
       end
 
       def disallowed_options

data/lib/sequenceserver/blast/report.rb

@@ -35,41 +35,72 @@ module SequenceServer
       # Attributes parsed from job metadata and BLAST output.
       attr_reader :querydb, :dbtype, :params
 
-      def to_json
-        [:querydb, :program, :program_version, :params, :stats,
-         :queries].inject({}) { |h, k|
+      def to_json(*_args)
+        generate
+
+        %i[querydb program program_version params stats
+           queries].inject({}) do |h, k|
           h[k] = send(k)
           h
-        }.update(search_id: job.id,
-                 submitted_at: job.submitted_at.utc,
-                 imported_xml: !!job.imported_xml_file,
-                 seqserv_version: SequenceServer::VERSION,
-                 non_parse_seqids: !!job.databases&.any?(&:non_parse_seqids?)).to_json
+        end.update(search_id: job.id,
+                   submitted_at: job.submitted_at.utc,
+                   imported_xml: !job.imported_xml_file.nil?,
+                   seqserv_version: SequenceServer::VERSION,
+                   cloud_sharing_enabled: SequenceServer.config[:cloud_share_url].start_with?('http'),
+                   non_parse_seqids: !!job.databases&.any?(&:non_parse_seqids?)).to_json
       end
 
-      private
+      def xml_file_size
+        return File.size(job.imported_xml_file) if job.imported_xml_file
+
+        generate
+
+        xml_formatter.size
+      end
 
       # Generate report.
       def generate
+        return self if @_generated
+
         job.raise!
         xml_ir = nil
         tsv_ir = nil
         if job.imported_xml_file
           xml_ir = parse_xml File.read(job.imported_xml_file)
-          tsv_ir = Hash.new do |h1,k1|
-            h1[k1] = Hash.new do |h2,k2|
-              h2[k2]=['','',[]]
+          tsv_ir = Hash.new do |h1, k1|
+            h1[k1] = Hash.new do |h2, k2|
+              h2[k2] = ['', '', []]
             end
           end
         else
-          xml_ir = parse_xml File.read(Formatter.run(job, 'xml').file)
-          tsv_ir = parse_tsv File.read(Formatter.run(job, 'custom_tsv').file)
+          xml_ir = parse_xml(xml_formatter.read_file)
+          tsv_ir = parse_tsv(tsv_formatter.read_file)
         end
         extract_program_info xml_ir
         extract_db_info xml_ir
         extract_params xml_ir
         extract_stats xml_ir
         extract_queries xml_ir, tsv_ir
+
+        @_generated = true
+
+        self
+      end
+
+      def done?
+        return true if job.imported_xml_file
+
+        File.exist?(xml_formatter.filepath) && File.exist?(tsv_formatter.filepath)
+      end
+
+      private
+
+      def xml_formatter
+        @xml_formatter ||= Formatter.run(job, 'xml')
+      end
+
+      def tsv_formatter
+        @tsv_formatter ||= Formatter.run(job, 'custom_tsv')
       end
 
       # Make program name and program name + version available via `program`
@@ -122,13 +153,13 @@ module SequenceServer
       def extract_stats(ir)
         stats  = ir[8].first[5][0]
         @stats = {
-          nsequences:   stats[0],
-          ncharacters:  stats[1],
-          hsp_length:   stats[2],
+          nsequences: stats[0],
+          ncharacters: stats[1],
+          hsp_length: stats[2],
           search_space: stats[3],
-          kappa:        stats[4],
-          labmda:       stats[5],
-          entropy:      stats[6]
+          kappa: stats[4],
+          labmda: stats[5],
+          entropy: stats[6]
         }
       end
 
@@ -144,6 +175,7 @@ module SequenceServer
       # Create Hit objects for the given query from the given ir.
       def extract_hits(xml_ir, tsv_ir, query)
         return if xml_ir == ["\n"] # => No hits.
+
         xml_ir.each do |n|
           # If hit comes from a non -parse_seqids database, then id (n[1]) is a
           # BLAST assigned internal id of the format 'gnl|BL_ORD_ID|serial'. We
@@ -169,7 +201,7 @@ module SequenceServer
       def extract_hsps(xml_ir, tsv_ir, hit)
         xml_ir.each_with_index do |n, i|
           n.insert(14, tsv_ir[i])
-          hsp = HSP.new(*[hit, *n])
+          hsp = HSP.new(hit, *n)
           hit.hsps << hsp
         end
       end
@@ -177,8 +209,9 @@ module SequenceServer
       def parse_xml(xml)
         node_to_array Ox.parse(xml).root
       rescue Ox::ParseError
-        fail 'Error parsing XML file' if job.imported_xml_file
-        fail InputError, <<~MSG
+        raise 'Error parsing XML file' if job.imported_xml_file
+
+        raise InputError, <<~MSG
           BLAST generated incorrect XML output. This can happen if sequence ids in your
           databases are not unique across all files. As a temporary workaround, you can
           repeat the search with one database at a time. Proper fix is to recreate the
@@ -231,9 +264,12 @@ module SequenceServer
       #    ...
       # }
       def parse_tsv(tsv)
-        ir = Hash.new {|h, k| h[k] = {} }
+        ir = Hash.new { |h, k| h[k] = {} }
         tsv.each_line do |line|
-          next if line.start_with? '#'; row = line.chomp.split("\t")
+          next if line.start_with? '#'
+
+          row = line.chomp.split("\t")
+
           (ir[row[0]][row[1]] ||= [row[2], row[3], []])[2] << row[4]
         end
         ir
@@ -246,14 +282,14 @@ module SequenceServer
 
         param_list.each_with_index do |word, i|
           nxt = param_list[i + 1]
-          if word.start_with? '-'
-            word.sub!('-', '')
-            unless nxt.nil? || nxt.start_with?('-')
-              res[word] = nxt
-            else
-              res[word] = 'True'
-            end
-          end
+          next unless word.start_with? '-'
+
+          word.sub!('-', '')
+          res[word] = unless nxt.nil? || nxt.start_with?('-')
+                        nxt
+                      else
+                        'True'
+                      end
         end
         res
       end

data/lib/sequenceserver/config.rb

@@ -11,12 +11,19 @@ module SequenceServer
 
     def initialize(data = {})
       @data = normalize data
+
       @config_file = @data.delete(:config_file)
       if @config_file
         @config_file = File.expand_path(@config_file)
-        @data = parse_config_file.update @data
+        @data = parse_config_file.deep_merge @data
       end
-      @data = defaults.update @data
+
+      @data = defaults.deep_merge @data
+
+      return unless @upgraded
+
+      logger.info 'You are using old configuration syntax. ' \
+                  'Run `sequenceserver -s` to update your config file syntax.'
     end
 
     attr_reader :data, :config_file
@@ -39,6 +46,7 @@ module SequenceServer
     # Write config data to config file.
     def write_config_file
       return unless config_file
+
       File.open(config_file, 'w') do |f|
         f.puts(data.delete_if { |_, v| v.nil? }.to_yaml)
       end
@@ -46,32 +54,34 @@ module SequenceServer
 
     private
 
-    # Symbolizes keys. Changes `database` key to `database_dir`.
+    # Symbolizes keys. Rename/reformat key-values.
     def normalize(data)
       return {} unless data
 
       # Symbolize keys.
-      data = Hash[data.map { |k, v| [k.to_sym, v] }]
+      data = data.transform_keys(&:to_sym)
 
-      # The newer config file version replaces the older database key with
-      # database_dir for correctness. Let's honour the old version as well.
+      # Very old config files may have a key called `database`.
+      # Rename it to `database_dir`
       if data[:database]
         database_dir = data.delete(:database)
         data[:database_dir] ||= database_dir
+        @upgrade = true
       end
 
-      # We would like a persistent :options: key in the config file, explicitly
-      # stating the parameters to use. Recommended values are written to config
-      # file on the first run. However, existing users won't have :options: key
-      # available from before. For them, we retain the old behaviour of
-      # automatically adding `-task blastn` for BLASTN searches.
-      if blast_opts = data.dig(:options, :blastn)
-        unless blast_opts.join.match('-task')
-          # Issue a warning.
-          logger.info "BLASTN will be run using '-task blastn' option." +
-                      " You can override this through configuration file."
-          data[:options][:blastn].push '-task blastn'
-        end
+      # Old config files may have an options hash with array values. We turn the
+      # array values into a hash. The logic is simple: If the array value is the
+      # same as default, we give it the key 'default', otherwise we give it the
+      # key 'custom'
+      data[:options]&.each do |key, val|
+        next if val.is_a? Hash
+
+        data[:options][key] = if val == defaults[:options][key][:default]
+                                { default: val }
+                              else
+                                { custom: val }
+                              end
+        @upgraded = true
       end
 
       data
@@ -86,8 +96,8 @@ module SequenceServer
       end
       logger.info "Reading configuration file: #{config_file}."
       normalize YAML.load_file(config_file)
-    rescue => error
-      raise CONFIG_FILE_ERROR.new(config_file, error)
+    rescue StandardError => e
+      raise CONFIG_FILE_ERROR.new(config_file, e)
     end
 
     def file?(file)
@@ -96,20 +106,35 @@ module SequenceServer
 
     # Default configuration data.
     def defaults
-      {
+      @defaults ||= {
         host: '0.0.0.0',
         port: 4567,
         databases_widget: 'classic',
         options: {
-          blastn:  ['-task blastn', '-evalue 1e-5'],
-          blastp:  ['-evalue 1e-5'],
-          blastx:  ['-evalue 1e-5'],
-          tblastx: ['-evalue 1e-5'],
-          tblastn: ['-evalue 1e-5']
+          blastn: {
+            default: ['-task blastn', '-evalue 1e-5']
+          },
+          blastp: {
+            default: ['-evalue 1e-5']
+          },
+          blastx: {
+            default: ['-evalue 1e-5']
+          },
+          tblastx: {
+            default: ['-evalue 1e-5']
+          },
+          tblastn: {
+            default: ['-evalue 1e-5']
+          }
         },
         num_threads: 1,
         num_jobs: 1,
-        job_lifetime: 43200
+        job_lifetime: 43_200,
+        # Set cloud_share_url to 'disabled' to disable the cloud sharing feature
+        cloud_share_url: 'https://share.sequenceserver.com/api/v1/shared-job',
+        # Warn in the UI before rendering results larger than this value
+        large_result_warning_threshold: 250 * 1024 * 1024,
+        optimistic: false # Faster, but does not perform DB compatibility checks
       }
     end
   end

data/lib/sequenceserver/job.rb

@@ -25,15 +25,19 @@ module SequenceServer
       # Creates and queues a job. Returns created job object.
       def create(params)
         job = BLAST::Job.new(params) # TODO: Dynamic dispatch.
-        pool.queue { job.run }
-        job
+        enqueue(job)
       end
 
       # Fetches job with the given id.
       def fetch(id)
         job_file = File.join(DOTDIR, id, 'job.yaml')
         fail NotFound unless File.exist?(job_file)
-        YAML.load_file(job_file)
+        if RUBY_VERSION < '3.1.0'
+          YAML.load_file(job_file)
+        else
+          YAML.load_file(job_file, permitted_classes: [Time, Symbol, SequenceServer::BLAST::Job, \
+                                                       SequenceServer::Database, SequenceServer::Job])
+        end
       end
 
       # Deletes job with the given id.
@@ -47,6 +51,12 @@ module SequenceServer
           .map { |f| fetch File.basename File.dirname f }
       end
 
+      # Enqueues a job that is already created, returns the job object
+      def enqueue(job)
+        pool.queue { job.run }
+        job
+      end
+
       private
 
       # Thread pool used for running BLAST searches.

data/lib/sequenceserver/job_remover.rb

@@ -1,18 +1,18 @@
 require 'sequenceserver/job'
 
 module SequenceServer
-
   # Removes expired jobs in a background thread.
   #
   # Job lifetime is provided in minutes. It can be zero in which case jobs will
   # be deleted as soon as they are done, or it can be Infinity in which case
   # jobs will never be deleted. Default is to delete finished jobs after 7 days.
   class JobRemover
-    DEFAULT_JOB_LIFETIME = 43200 # minutes (i.e., 30 days)
+    DEFAULT_JOB_LIFETIME = 43_200 # minutes (i.e., 30 days)
 
     def initialize(job_lifetime)
       @job_lifetime = job_lifetime || DEFAULT_JOB_LIFETIME
-      return if @job_lifetime == Float::INFINITY
+      return if @job_lifetime == 'Infinity' || @job_lifetime == Float::INFINITY
+
       @job_lifetime = Integer(@job_lifetime) * 60
       spawn_cleanup
     end
@@ -25,34 +25,30 @@ module SequenceServer
     def spawn_cleanup
       Thread.new do
         loop do
-          begin
-            now = Time.now
-            finished_jobs = Job.all.select { |f| f.done? }
-
-            # Cleanup finished jobs that have lived a lifetime.
-            expired_jobs = finished_jobs.select do |job|
-              (job.completed_at + @job_lifetime) <= now
-            end
-            expired_jobs.each { |job| Job.delete job.id }
-            unless expired_jobs.empty?
-              logger.info "#{logid}: deleted #{expired_jobs.count} old jobs."
-            end
-
-            # Decide when to cleanup next.
-            remaining_jobs = finished_jobs - expired_jobs
-            oldest_time = remaining_jobs.map(&:completed_at).min
-            if oldest_time && now < (oldest_time + @job_lifetime)
-              @next_cleanup = (oldest_time + @job_lifetime) - now
-            else
-              @next_cleanup = @job_lifetime
-            end
+          now = Time.now
+          finished_jobs = Job.all.select(&:done?)
 
-            # Sleep till next cleanup.
-            sleep @next_cleanup.ceil
-          rescue => e
-            logger.fatal("#{logid}: #{e.inspect}\n#{e.backtrace.join("\n")}")
-            Thread.exit
+          # Cleanup finished jobs that have lived a lifetime.
+          expired_jobs = finished_jobs.select do |job|
+            (job.completed_at + @job_lifetime) <= now
           end
+          expired_jobs.each { |job| Job.delete job.id }
+          logger.info "#{logid}: deleted #{expired_jobs.count} old jobs." unless expired_jobs.empty?
+
+          # Decide when to cleanup next.
+          remaining_jobs = finished_jobs - expired_jobs
+          oldest_time = remaining_jobs.map(&:completed_at).min
+          @next_cleanup = if oldest_time && now < (oldest_time + @job_lifetime)
+                            (oldest_time + @job_lifetime) - now
+                          else
+                            @job_lifetime
+                          end
+
+          # Sleep till next cleanup.
+          sleep @next_cleanup.ceil
+        rescue StandardError => e
+          logger.fatal("#{logid}: #{e.inspect}\n#{e.backtrace.join("\n")}")
+          Thread.exit
         end
       end
     end