sequenceserver 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25c2424857780517682276d21694f8ab0c3594772b82f08dcdca1934dcf23fd9
-  data.tar.gz: 3e27329d829825b7a3a73f14ebdecad514648381dbba914cfca6ee63564c795a
+  metadata.gz: 3cf703fae66ad96d285ed0556c387dabb0d69be120db5bdd3c3f9f8a03d3a814
+  data.tar.gz: 503bb0b69cbf773ab536d99f6782f4fff19f4872c49e6377d49a9f7820861c3b
 SHA512:
-  metadata.gz: 5137bf6118ee0044a0b8d5b1979f525686d7dd1698d6c795682d39a6c9d8247410104441fb642a585e6df9a18ad827336c8f61215f9e3c8c68fb387e0b6e2f41
-  data.tar.gz: e7ec5cebd2595b3a5f90cb750ce583161821e1be22dfcf14b1ba1fbe11dda440105795ac04b06caeb456c0f78e55db50d730704db8972667ca8d7acae5170692
+  metadata.gz: 1896b2afddf6033c98747f7eba1dabbe15b270940bd19760ebef4ed75af267760d45d40215663173ccb2f964472b458179a459316b86ef72bc26aa75d62faad0
+  data.tar.gz: 4ac1d9a5a205d299af4b1467521a9bc539e3833d0a571cfa90b490b19df958fc0f7ee2e1341e3d8d5e19fccbe2ee6874884fc7fbcbfbdc6abbe1e60df3c9a76e

data/lib/sequenceserver/api_errors.rb

@@ -1,8 +1,30 @@
 module SequenceServer
+  Error = Class.new(StandardError)
+
+  # DatabaseUnreachableError is raised when the serialised Job object is
+  # refering to a database that is not present in the current filesystem.
+  class DatabaseUnreachableError < Error
+    attr_reader :more_info
+
+    def initialize(more_info)
+      super
+      @more_info = more_info
+    end
+
+    def title
+      'Sequence database unreachable'
+    end
+
+    def message
+      "The action you're trying to perform is not possible because \
+        the database is unreachable. This can happen if the database has \
+        been deleted or you are performing an action on an imported job."
+    end
+  end
+
   # API errors have an http status, title, message, and additional information
   # like stacktrace or information from program output.
-  class APIError < StandardError
-  end
+  APIError = Class.new(Error)
 
   # Job not found (404).
   class NotFound < APIError

data/lib/sequenceserver/blast/job.rb

@@ -1,4 +1,5 @@
 require 'sequenceserver/job'
+require 'sequenceserver/zip_file_generator'
 
 module SequenceServer
   # BLAST module.
@@ -24,13 +25,18 @@ module SequenceServer
             @databases = Database[params[:databases]]
             @advanced  = params[:advanced].to_s.strip
             @options   = @advanced + defaults
+            # The following params are for analytics only
+            @num_threads = config[:num_threads]
+            @query_length = calculate_query_size
+            @databases_ncharacters_total = calculate_databases_ncharacters_total
           end
         end
       end
 
       # :nodoc:
       # Attributes used by us - should be considered private.
-      attr_reader :method, :qfile, :databases, :advanced, :options
+      attr_reader :advanced
+      attr_reader :databases, :databases_ncharacters_total, :method, :num_threads, :options, :qfile, :query_length
 
       # :nodoc:
       # Deprecated; see Report#extract_params
@@ -53,7 +59,6 @@ module SequenceServer
       # Override Job#raise! to raise specific API errors based on exitstatus
       # and using contents of stderr to provide context about the error.
       #
-      # rubocop:disable Metrics/CyclomaticComplexity
       def raise!
         # Return true exit status is 0 and stdout is not empty.
         return true if exitstatus.zero? && !File.zero?(stdout)
@@ -92,10 +97,38 @@ module SequenceServer
           MSG
         end
       end
-      # rubocop:enable Metrics/CyclomaticComplexity
+
+      # Use it with a block to get a self-cleaning temporary archive file
+      # of the contents of the job directory.
+      # job.as_archived_file do |tmp_file|
+      #    # do things with tmp_file
+      # end
+      def as_archived_file(&block)
+        Dir.mktmpdir(id.to_s) do |tmp_dir|
+          file_path = "#{tmp_dir}/#{id}.zip"
+          ZipFileGenerator.new(dir, file_path).write
+          File.open(file_path, 'r') do |file|
+            block.call(file)
+          end
+        end
+      end
 
       private
 
+      def calculate_databases_ncharacters_total
+        databases.map(&:ncharacters).map(&:to_i).reduce(:+)
+      end
+
+      def calculate_query_size
+        size = 0
+        IO.foreach(@qfile) do |line|
+          next if line[0] == '>'
+
+          size += line.gsub(/\s+/, '').length
+        end
+        size
+      end
+
       def validate(params)
         validate_method params[:method]
         validate_sequences params[:sequence]
@@ -109,12 +142,14 @@ module SequenceServer
 
       def validate_method(method)
         return true if ALGORITHMS.include? method
+
         fail InputError, 'BLAST algorithm should be one of:' \
                             " #{ALGORITHMS.join(', ')}."
       end
 
       def validate_sequences(sequences)
         return true if sequences.is_a?(String) && !sequences.empty?
+
         fail InputError, 'Sequences should be a non-empty string.'
       end
 
@@ -122,6 +157,7 @@ module SequenceServer
         ids = Database.ids
         return true if database_ids.is_a?(Array) && !database_ids.empty? &&
                        (ids & database_ids).length == database_ids.length
+
         fail InputError, "Database id should be one of: #{ids.join("\n")}."
       end
 
@@ -133,9 +169,7 @@ module SequenceServer
         return true if !options || (options.is_a?(String) &&
                                     options.strip.empty?)
 
-        unless allowed_chars.match(options)
-          fail InputError, 'Invalid characters detected in options.'
-        end
+        fail InputError, 'Invalid characters detected in options.' unless allowed_chars.match(options)
 
         if disallowed_options.match(options)
           failedopt = Regexp.last_match[0]
@@ -146,7 +180,7 @@ module SequenceServer
       end
 
       def allowed_chars
-        /\A[a-z0-9\-_\. ',]*\Z/i
+        /\A[a-z0-9\-_. ',]*\Z/i
       end
 
       def disallowed_options

data/lib/sequenceserver/blast/report.rb

@@ -35,16 +35,17 @@ module SequenceServer
       # Attributes parsed from job metadata and BLAST output.
       attr_reader :querydb, :dbtype, :params
 
-      def to_json
-        [:querydb, :program, :program_version, :params, :stats,
-         :queries].inject({}) { |h, k|
+      def to_json(*_args)
+        %i[querydb program program_version params stats
+           queries].inject({}) do |h, k|
           h[k] = send(k)
           h
-        }.update(search_id: job.id,
-                 submitted_at: job.submitted_at.utc,
-                 imported_xml: !!job.imported_xml_file,
-                 seqserv_version: SequenceServer::VERSION,
-                 non_parse_seqids: !!job.databases&.any?(&:non_parse_seqids?)).to_json
+        end.update(search_id: job.id,
+                   submitted_at: job.submitted_at.utc,
+                   imported_xml: !job.imported_xml_file.nil?,
+                   seqserv_version: SequenceServer::VERSION,
+                   cloud_sharing_enabled: SequenceServer.config[:cloud_share_url].start_with?('http'),
+                   non_parse_seqids: !!job.databases&.any?(&:non_parse_seqids?)).to_json
       end
 
       private
@@ -56,9 +57,9 @@ module SequenceServer
         tsv_ir = nil
         if job.imported_xml_file
           xml_ir = parse_xml File.read(job.imported_xml_file)
-          tsv_ir = Hash.new do |h1,k1|
-            h1[k1] = Hash.new do |h2,k2|
-              h2[k2]=['','',[]]
+          tsv_ir = Hash.new do |h1, k1|
+            h1[k1] = Hash.new do |h2, k2|
+              h2[k2] = ['', '', []]
             end
           end
         else
@@ -122,13 +123,13 @@ module SequenceServer
       def extract_stats(ir)
         stats  = ir[8].first[5][0]
         @stats = {
-          nsequences:   stats[0],
-          ncharacters:  stats[1],
-          hsp_length:   stats[2],
+          nsequences: stats[0],
+          ncharacters: stats[1],
+          hsp_length: stats[2],
           search_space: stats[3],
-          kappa:        stats[4],
-          labmda:       stats[5],
-          entropy:      stats[6]
+          kappa: stats[4],
+          labmda: stats[5],
+          entropy: stats[6]
         }
       end
 
@@ -144,6 +145,7 @@ module SequenceServer
       # Create Hit objects for the given query from the given ir.
       def extract_hits(xml_ir, tsv_ir, query)
         return if xml_ir == ["\n"] # => No hits.
+
         xml_ir.each do |n|
           # If hit comes from a non -parse_seqids database, then id (n[1]) is a
           # BLAST assigned internal id of the format 'gnl|BL_ORD_ID|serial'. We
@@ -169,7 +171,7 @@ module SequenceServer
       def extract_hsps(xml_ir, tsv_ir, hit)
         xml_ir.each_with_index do |n, i|
           n.insert(14, tsv_ir[i])
-          hsp = HSP.new(*[hit, *n])
+          hsp = HSP.new(hit, *n)
           hit.hsps << hsp
         end
       end
@@ -177,8 +179,9 @@ module SequenceServer
       def parse_xml(xml)
         node_to_array Ox.parse(xml).root
       rescue Ox::ParseError
-        fail 'Error parsing XML file' if job.imported_xml_file
-        fail InputError, <<~MSG
+        raise 'Error parsing XML file' if job.imported_xml_file
+
+        raise InputError, <<~MSG
           BLAST generated incorrect XML output. This can happen if sequence ids in your
           databases are not unique across all files. As a temporary workaround, you can
           repeat the search with one database at a time. Proper fix is to recreate the
@@ -231,9 +234,12 @@ module SequenceServer
       #    ...
       # }
       def parse_tsv(tsv)
-        ir = Hash.new {|h, k| h[k] = {} }
+        ir = Hash.new { |h, k| h[k] = {} }
         tsv.each_line do |line|
-          next if line.start_with? '#'; row = line.chomp.split("\t")
+          next if line.start_with? '#'
+
+          row = line.chomp.split("\t")
+
           (ir[row[0]][row[1]] ||= [row[2], row[3], []])[2] << row[4]
         end
         ir
@@ -246,14 +252,14 @@ module SequenceServer
 
         param_list.each_with_index do |word, i|
           nxt = param_list[i + 1]
-          if word.start_with? '-'
-            word.sub!('-', '')
-            unless nxt.nil? || nxt.start_with?('-')
-              res[word] = nxt
-            else
-              res[word] = 'True'
-            end
-          end
+          next unless word.start_with? '-'
+
+          word.sub!('-', '')
+          res[word] = unless nxt.nil? || nxt.start_with?('-')
+                        nxt
+                      else
+                        'True'
+                      end
         end
         res
       end

data/lib/sequenceserver/config.rb

@@ -11,12 +11,19 @@ module SequenceServer
 
     def initialize(data = {})
       @data = normalize data
+
       @config_file = @data.delete(:config_file)
       if @config_file
         @config_file = File.expand_path(@config_file)
-        @data = parse_config_file.update @data
+        @data = parse_config_file.deep_merge @data
       end
-      @data = defaults.update @data
+
+      @data = defaults.deep_merge @data
+
+      return unless @upgraded
+
+      logger.info 'You are using old configuration syntax. ' \
+                  'Run `sequenceserver -s` to update your config file syntax.'
     end
 
     attr_reader :data, :config_file
@@ -39,6 +46,7 @@ module SequenceServer
     # Write config data to config file.
     def write_config_file
       return unless config_file
+
       File.open(config_file, 'w') do |f|
         f.puts(data.delete_if { |_, v| v.nil? }.to_yaml)
       end
@@ -46,32 +54,34 @@ module SequenceServer
 
     private
 
-    # Symbolizes keys. Changes `database` key to `database_dir`.
+    # Symbolizes keys. Rename/reformat key-values.
     def normalize(data)
       return {} unless data
 
       # Symbolize keys.
-      data = Hash[data.map { |k, v| [k.to_sym, v] }]
+      data = data.transform_keys(&:to_sym)
 
-      # The newer config file version replaces the older database key with
-      # database_dir for correctness. Let's honour the old version as well.
+      # Very old config files may have a key called `database`.
+      # Rename it to `database_dir`
       if data[:database]
         database_dir = data.delete(:database)
         data[:database_dir] ||= database_dir
+        @upgrade = true
       end
 
-      # We would like a persistent :options: key in the config file, explicitly
-      # stating the parameters to use. Recommended values are written to config
-      # file on the first run. However, existing users won't have :options: key
-      # available from before. For them, we retain the old behaviour of
-      # automatically adding `-task blastn` for BLASTN searches.
-      if blast_opts = data.dig(:options, :blastn)
-        unless blast_opts.join.match('-task')
-          # Issue a warning.
-          logger.info "BLASTN will be run using '-task blastn' option." +
-                      " You can override this through configuration file."
-          data[:options][:blastn].push '-task blastn'
-        end
+      # Old config files may have an options hash with array values. We turn the
+      # array values into a hash. The logic is simple: If the array value is the
+      # same as default, we give it the key 'default', otherwise we give it the
+      # key 'custom'
+      data[:options]&.each do |key, val|
+        next if val.is_a? Hash
+
+        data[:options][key] = if val == defaults[:options][key][:default]
+                                { default: val }
+                              else
+                                { custom: val }
+                              end
+        @upgraded = true
       end
 
       data
@@ -86,8 +96,8 @@ module SequenceServer
       end
       logger.info "Reading configuration file: #{config_file}."
       normalize YAML.load_file(config_file)
-    rescue => error
-      raise CONFIG_FILE_ERROR.new(config_file, error)
+    rescue StandardError => e
+      raise CONFIG_FILE_ERROR.new(config_file, e)
     end
 
     def file?(file)
@@ -96,20 +106,32 @@ module SequenceServer
 
     # Default configuration data.
     def defaults
-      {
+      @defaults ||= {
         host: '0.0.0.0',
         port: 4567,
         databases_widget: 'classic',
         options: {
-          blastn:  ['-task blastn', '-evalue 1e-5'],
-          blastp:  ['-evalue 1e-5'],
-          blastx:  ['-evalue 1e-5'],
-          tblastx: ['-evalue 1e-5'],
-          tblastn: ['-evalue 1e-5']
+          blastn: {
+            default: ['-task blastn', '-evalue 1e-5']
+          },
+          blastp: {
+            default: ['-evalue 1e-5']
+          },
+          blastx: {
+            default: ['-evalue 1e-5']
+          },
+          tblastx: {
+            default: ['-evalue 1e-5']
+          },
+          tblastn: {
+            default: ['-evalue 1e-5']
+          }
         },
         num_threads: 1,
         num_jobs: 1,
-        job_lifetime: 43200
+        job_lifetime: 43_200,
+        # Set cloud_share_url to 'disabled' to disable the cloud sharing feature
+        cloud_share_url: 'https://share.sequenceserver.com/api/v1/shared-job'
       }
     end
   end

data/lib/sequenceserver/job.rb

@@ -25,15 +25,19 @@ module SequenceServer
       # Creates and queues a job. Returns created job object.
       def create(params)
         job = BLAST::Job.new(params) # TODO: Dynamic dispatch.
-        pool.queue { job.run }
-        job
+        enqueue(job)
       end
 
       # Fetches job with the given id.
       def fetch(id)
         job_file = File.join(DOTDIR, id, 'job.yaml')
         fail NotFound unless File.exist?(job_file)
-        YAML.load_file(job_file)
+        if RUBY_VERSION < '3.1.0'
+          YAML.load_file(job_file)
+        else
+          YAML.load_file(job_file, permitted_classes: [Time, Symbol, SequenceServer::BLAST::Job, \
+                                                       SequenceServer::Database, SequenceServer::Job])
+        end
       end
 
       # Deletes job with the given id.
@@ -47,6 +51,12 @@ module SequenceServer
           .map { |f| fetch File.basename File.dirname f }
       end
 
+      # Enqueues a job that is already created, returns the job object
+      def enqueue(job)
+        pool.queue { job.run }
+        job
+      end
+
       private
 
       # Thread pool used for running BLAST searches.

data/lib/sequenceserver/job_remover.rb

@@ -1,18 +1,18 @@
 require 'sequenceserver/job'
 
 module SequenceServer
-
   # Removes expired jobs in a background thread.
   #
   # Job lifetime is provided in minutes. It can be zero in which case jobs will
   # be deleted as soon as they are done, or it can be Infinity in which case
   # jobs will never be deleted. Default is to delete finished jobs after 7 days.
   class JobRemover
-    DEFAULT_JOB_LIFETIME = 43200 # minutes (i.e., 30 days)
+    DEFAULT_JOB_LIFETIME = 43_200 # minutes (i.e., 30 days)
 
     def initialize(job_lifetime)
       @job_lifetime = job_lifetime || DEFAULT_JOB_LIFETIME
-      return if @job_lifetime == Float::INFINITY
+      return if @job_lifetime == 'Infinity' || @job_lifetime == Float::INFINITY
+
       @job_lifetime = Integer(@job_lifetime) * 60
       spawn_cleanup
     end
@@ -25,34 +25,30 @@ module SequenceServer
     def spawn_cleanup
       Thread.new do
         loop do
-          begin
-            now = Time.now
-            finished_jobs = Job.all.select { |f| f.done? }
-
-            # Cleanup finished jobs that have lived a lifetime.
-            expired_jobs = finished_jobs.select do |job|
-              (job.completed_at + @job_lifetime) <= now
-            end
-            expired_jobs.each { |job| Job.delete job.id }
-            unless expired_jobs.empty?
-              logger.info "#{logid}: deleted #{expired_jobs.count} old jobs."
-            end
-
-            # Decide when to cleanup next.
-            remaining_jobs = finished_jobs - expired_jobs
-            oldest_time = remaining_jobs.map(&:completed_at).min
-            if oldest_time && now < (oldest_time + @job_lifetime)
-              @next_cleanup = (oldest_time + @job_lifetime) - now
-            else
-              @next_cleanup = @job_lifetime
-            end
+          now = Time.now
+          finished_jobs = Job.all.select(&:done?)
 
-            # Sleep till next cleanup.
-            sleep @next_cleanup.ceil
-          rescue => e
-            logger.fatal("#{logid}: #{e.inspect}\n#{e.backtrace.join("\n")}")
-            Thread.exit
+          # Cleanup finished jobs that have lived a lifetime.
+          expired_jobs = finished_jobs.select do |job|
+            (job.completed_at + @job_lifetime) <= now
           end
+          expired_jobs.each { |job| Job.delete job.id }
+          logger.info "#{logid}: deleted #{expired_jobs.count} old jobs." unless expired_jobs.empty?
+
+          # Decide when to cleanup next.
+          remaining_jobs = finished_jobs - expired_jobs
+          oldest_time = remaining_jobs.map(&:completed_at).min
+          @next_cleanup = if oldest_time && now < (oldest_time + @job_lifetime)
+                            (oldest_time + @job_lifetime) - now
+                          else
+                            @job_lifetime
+                          end
+
+          # Sleep till next cleanup.
+          sleep @next_cleanup.ceil
+        rescue StandardError => e
+          logger.fatal("#{logid}: #{e.inspect}\n#{e.backtrace.join("\n")}")
+          Thread.exit
         end
       end
     end

data/lib/sequenceserver/refinements.rb

@@ -0,0 +1,15 @@
+class Hash
+  def deep_copy
+    Marshal.load Marshal.dump self
+  end
+
+  def deep_merge(other)
+    merge(other) do |key, oldval, newval|
+      if oldval.is_a? Hash then
+        oldval.deep_merge(newval)
+      else
+        newval
+      end
+    end
+  end
+end

data/lib/sequenceserver/routes.rb

@@ -1,6 +1,7 @@
 require 'json'
 require 'tilt/erb'
 require 'sinatra/base'
+require 'rest-client'
 
 require 'sequenceserver/job'
 require 'sequenceserver/blast'
@@ -79,9 +80,7 @@ module SequenceServer
         options: SequenceServer.config[:options]
       }
 
-      if SequenceServer.config[:databases_widget] == 'tree'
-        searchdata.update(tree: Database.tree)
-      end
+      searchdata.update(tree: Database.tree) if SequenceServer.config[:databases_widget] == 'tree'
 
       # If a job_id is specified, update searchdata from job meta data (i.e.,
       # query, pre-selected databases, advanced options used). Query is only
@@ -138,7 +137,7 @@ module SequenceServer
       database_ids = params['database_ids'].split(',')
       sequences = Sequence::Retriever.new(sequence_ids, database_ids, true)
       send_file(sequences.file.path,
-                type:     sequences.mime,
+                type: sequences.mime,
                 filename: sequences.filename)
     end
 
@@ -149,6 +148,57 @@ module SequenceServer
       send_file out.file, filename: out.filename, type: out.mime
     end
 
+    post '/cloud_share' do
+      content_type :json
+      request_params = JSON.parse(request.body.read)
+      job = Job.fetch(request_params['job_id'])
+
+      unless job.done?
+        status 422
+        { errors: ["Job #{request_params['job_id']} is not finished yet."] }.to_json
+      end
+
+      unless SequenceServer.config[:cloud_share_url]
+        status 503
+        { errors: ['Sorry, cloud sharing is not enabled on this server.'] }.to_json
+      end
+
+      begin
+        job.as_archived_file do |archived_job_file|
+          cloud_share_response = RestClient.post(
+            SequenceServer.config[:cloud_share_url],
+            {
+              shared_job: {
+                sender: {
+                  email: request_params['sender_email']
+                },
+                archived_job_file: archived_job_file,
+                original_job_id: job.id
+              }
+            }
+          )
+
+          return cloud_share_response.body
+        end
+      rescue RestClient::ExceptionWithResponse => e
+        cloud_share_response = e.response
+
+        case cloud_share_response.code
+        when 413
+          halt 413,
+               { errors: ['Sorry, the results are too large to share, please consider \
+                  using https://sequenceserver.com/cloud'] }.to_json
+        when 422
+          halt 422, JSON.parse(cloud_share_response.body).to_json
+        else
+          error cloud_share_response.code,
+                { errors: ["Unexpected Cloudshare response: #{cloud_share_response.code}"] }.to_json
+        end
+      rescue Errno::ECONNREFUSED
+        error 503, { errors: ['Sorry, the cloud sharing server may not be running. Try again later.'] }.to_json
+      end
+    end
+
     # Catches any exception raised within the app and returns JSON
     # representation of the error:
     # {
@@ -173,6 +223,7 @@ module SequenceServer
     # more_info.
     error 400..500 do
       error = env['sinatra.error']
+      return unless error
 
       # All errors will have a message.
       error_data = { message: error.message }
@@ -193,7 +244,13 @@ module SequenceServer
         error_data[:more_info] = error.backtrace.join("\n")
       end
 
-      error_data.to_json
+      if request.env['HTTP_ACCEPT'].to_s.include?('application/json')
+        content_type :json
+        error_data.to_json
+      else
+        content_type :html
+        erb :error, locals: { error_data: error_data }, layout: true
+      end
     end
 
     # Get the query sequences, selected databases, and advanced params used.
@@ -202,7 +259,7 @@ module SequenceServer
       return if job.imported_xml_file
 
       # Only read job.qfile if we are not going to use Database.retrieve.
-      searchdata[:query] = File.read(job.qfile) if !params[:query]
+      searchdata[:query] = File.read(job.qfile) unless params[:query]
 
       # Which databases to pre-select.
       searchdata[:preSelectedDbs] = job.databases
@@ -213,7 +270,12 @@ module SequenceServer
       # default advanced parameters from the advanced params input field. In
       # this case, we do want the advanced params input field to be empty when
       # the user hits the back button. Thus we do not test for empty string.
-      searchdata[:options][job.method] = [job.advanced] if job.advanced
+      method = job.method.to_sym
+      if job.advanced && job.advanced !=
+                         searchdata[:options][method][:default].join(' ')
+        searchdata[:options] = searchdata[:options].deep_copy
+        searchdata[:options][method]['last search'] = [job.advanced]
+      end
     end
   end
 end