sequenceserver 2.0.0.rc8 → 2.1.0

data/lib/sequenceserver/database.rb

@@ -19,14 +19,14 @@ module SequenceServer
   # SequenceServer will always place BLAST database files alongside input FASTA,
   # and use `parse_seqids` option of `makeblastdb` to format databases.
   Database = Struct.new(:name, :title, :type, :nsequences, :ncharacters,
-                        :updated_on) do
+                        :updated_on, :format, :categories) do
 
     extend Forwardable
 
     def_delegators SequenceServer, :config, :sys
 
     def initialize(*args)
-      args[2].downcase! # database type
+      args[2].downcase!   # type
       args.each(&:freeze)
       super
 
@@ -34,6 +34,7 @@ module SequenceServer
     end
 
     attr_reader :id
+    alias path name
 
     def retrieve(accession, coords = nil)
       cmd = "blastdbcmd -db #{name} -entry '#{accession}'"
@@ -48,10 +49,36 @@ module SequenceServer
       nil
     end
 
-    def include?(accession)
-      cmd = "blastdbcmd -entry '#{accession}' -db #{name}"
-      out, = sys(cmd, path: config[:bin])
-      !out.empty?
+    # Returns true if the database contains the given sequence id.
+    # Returns false otherwise.
+    def include?(id)
+      cmd = "blastdbcmd -entry '#{id}' -db #{name}"
+      sys(cmd, path: config[:bin]) rescue false
+    end
+
+    def v4?
+      format == '4'
+    end
+
+    def v5?
+      format == '5'
+    end
+
+    # Return true if the database was _not_ created using the -parse_seqids
+    # option of makeblastdb.
+    def non_parse_seqids?
+      return if alias?
+      case format
+      when '5'
+        (%w[nog nos pog pos] & extensions).length != 2
+      when '4'
+        (%w[nog nsd nsi pod psd psi] & extensions).length != 3
+      end
+    end
+
+    # Returns true if the database was created using blastdb_aliastool.
+    def alias?
+      (%w[nal pal] & extensions).length == 1 && extensions.count == 1
     end
 
     def ==(other)
@@ -65,6 +92,16 @@ module SequenceServer
     def to_json(*args)
       to_h.update(id: id).to_json(*args)
     end
+
+    private
+
+    def extensions
+      # The glob pattern used here is quite relaxed. This is to capture
+      # multipart databases as well. It is possible that non-blast-database
+      # extensions may also be picked. However, that shouldn't be a problem
+      # as we only check whether certain required extensions are present or not.
+      @extensions ||= Dir["#{path}*{n,p}*"].map { |p| p.split('.').last }.sort.uniq
+    end
   end
 
   # Model Database's eigenclass as a collection of Database objects.
@@ -80,9 +117,8 @@ module SequenceServer
         @collection ||= {}
       end
 
-      def collection=(databases_attrs)
-        databases_attrs.each do |db_attrs|
-          db = Database.new(*db_attrs)
+      def collection=(databases)
+        databases.each do |db|
           collection[db.id] = db
         end
       end
@@ -102,6 +138,30 @@ module SequenceServer
         collection.values
       end
 
+      def tree
+        all.each_with_object({}) do |db, data|
+          data[db.type] ||= []
+          use_parent = '#'
+          db.categories.each_with_index do |entry, index|
+            parent = index.zero? ? '#' : db.categories[0..(index - 1)].join('-')
+            use_id = db.categories[0..index].join('-')
+            element = { id: use_id, parent: parent, text: entry }
+            data[db.type] << element unless data[db.type].include?(element)
+            use_parent = use_id
+          end
+
+          data[db.type] <<
+            {
+              id: db.id,
+              parent: use_parent,
+              text: db.title,
+              icon: 'glyphicon glyphicon-file'
+            }
+
+          yield(db, data[db.type].last) if block_given?
+        end
+      end
+
       def each(&block)
         all.each(&block)
       end

data/lib/sequenceserver/job.rb

@@ -25,15 +25,19 @@ module SequenceServer
       # Creates and queues a job. Returns created job object.
       def create(params)
         job = BLAST::Job.new(params) # TODO: Dynamic dispatch.
-        pool.queue { job.run }
-        job
+        enqueue(job)
       end
 
       # Fetches job with the given id.
       def fetch(id)
         job_file = File.join(DOTDIR, id, 'job.yaml')
         fail NotFound unless File.exist?(job_file)
-        YAML.load_file(job_file)
+        if RUBY_VERSION < '3.1.0'
+          YAML.load_file(job_file)
+        else
+          YAML.load_file(job_file, permitted_classes: [Time, Symbol, SequenceServer::BLAST::Job, \
+                                                       SequenceServer::Database, SequenceServer::Job])
+        end
       end
 
       # Deletes job with the given id.
@@ -47,11 +51,17 @@ module SequenceServer
           .map { |f| fetch File.basename File.dirname f }
       end
 
+      # Enqueues a job that is already created, returns the job object
+      def enqueue(job)
+        pool.queue { job.run }
+        job
+      end
+
       private
 
       # Thread pool used for running BLAST searches.
       def pool
-        @pool ||= Pool.new(SequenceServer.config[:num_threads])
+        @pool ||= Pool.new(SequenceServer.config[:num_jobs] || 1)
       end
     end
 

data/lib/sequenceserver/job_remover.rb

@@ -1,18 +1,18 @@
 require 'sequenceserver/job'
 
 module SequenceServer
-
   # Removes expired jobs in a background thread.
   #
   # Job lifetime is provided in minutes. It can be zero in which case jobs will
   # be deleted as soon as they are done, or it can be Infinity in which case
   # jobs will never be deleted. Default is to delete finished jobs after 7 days.
   class JobRemover
-    DEFAULT_JOB_LIFETIME = 43200 # minutes (i.e., 30 days)
+    DEFAULT_JOB_LIFETIME = 43_200 # minutes (i.e., 30 days)
 
     def initialize(job_lifetime)
       @job_lifetime = job_lifetime || DEFAULT_JOB_LIFETIME
-      return if @job_lifetime == Float::INFINITY
+      return if @job_lifetime == 'Infinity' || @job_lifetime == Float::INFINITY
+
       @job_lifetime = Integer(@job_lifetime) * 60
       spawn_cleanup
     end
@@ -25,34 +25,30 @@ module SequenceServer
     def spawn_cleanup
       Thread.new do
         loop do
-          begin
-            now = Time.now
-            finished_jobs = Job.all.select { |f| f.done? }
-
-            # Cleanup finished jobs that have lived a lifetime.
-            expired_jobs = finished_jobs.select do |job|
-              (job.completed_at + @job_lifetime) <= now
-            end
-            expired_jobs.each { |job| Job.delete job.id }
-            unless expired_jobs.empty?
-              logger.info "#{logid}: deleted #{expired_jobs.count} old jobs."
-            end
-
-            # Decide when to cleanup next.
-            remaining_jobs = finished_jobs - expired_jobs
-            oldest_time = remaining_jobs.map(&:completed_at).min
-            if oldest_time && now < (oldest_time + @job_lifetime)
-              @next_cleanup = (oldest_time + @job_lifetime) - now
-            else
-              @next_cleanup = @job_lifetime
-            end
+          now = Time.now
+          finished_jobs = Job.all.select(&:done?)
 
-            # Sleep till next cleanup.
-            sleep @next_cleanup.ceil
-          rescue => e
-            logger.fatal("#{logid}: #{e.inspect}\n#{e.backtrace.join("\n")}")
-            Thread.exit
+          # Cleanup finished jobs that have lived a lifetime.
+          expired_jobs = finished_jobs.select do |job|
+            (job.completed_at + @job_lifetime) <= now
           end
+          expired_jobs.each { |job| Job.delete job.id }
+          logger.info "#{logid}: deleted #{expired_jobs.count} old jobs." unless expired_jobs.empty?
+
+          # Decide when to cleanup next.
+          remaining_jobs = finished_jobs - expired_jobs
+          oldest_time = remaining_jobs.map(&:completed_at).min
+          @next_cleanup = if oldest_time && now < (oldest_time + @job_lifetime)
+                            (oldest_time + @job_lifetime) - now
+                          else
+                            @job_lifetime
+                          end
+
+          # Sleep till next cleanup.
+          sleep @next_cleanup.ceil
+        rescue StandardError => e
+          logger.fatal("#{logid}: #{e.inspect}\n#{e.backtrace.join("\n")}")
+          Thread.exit
         end
       end
     end

data/lib/sequenceserver/makeblastdb.rb

@@ -11,17 +11,6 @@ module SequenceServer
   #   makeblastdb.scan && makeblastdb.run
   #
   class MAKEBLASTDB
-    # We want V5 databases created using -parse_seqids for proper function of
-    # SequenceServer. This means each database should be comprised of at least 9
-    # files with the following extensions. Databases created by us will have two
-    # additional files with the extensions nhd and nhi, or phd and phi, due to
-    # the use of -hash_index option. Finally, multipart databases will have one
-    # additional file with the extension nal or pal.
-    REQUIRED_EXTENSIONS = {
-      'nucleotide' => %w{ndb nhr nin nog nos not nsq ntf nto}.freeze,
-      'protein' => %w{pdb phr pin pog pos pot psq ptf pto}.freeze
-    }
-
     extend Forwardable
 
     def_delegators SequenceServer, :config, :sys
@@ -70,18 +59,12 @@ module SequenceServer
     # Returns true if the databases directory contains one or more incompatible
     # databases.
     #
-    # Note that it is okay to only use V4 databases or only V5 databases. It is
+    # Note that it is okay to only use V4 databases or only V5 databases.
     # Incompatibility arises when they are mixed.
     def any_incompatible?
-      return false if @fastas_to_reformat.empty?
-      # We need to compare @fastas_to_reformat to @formatted_fastas, but the
-      # latter contains extra attributes. However, the first attribute, i.e,
-      # path is common to both and sufficient for our needs.
-      to_reformat = @fastas_to_reformat.map(&:first)
-      formatted = @formatted_fastas.map(&:first)
-      # Check that they are not equal. Using intersection operator ensures
-      # comparison even if their order differs.
-      formatted & to_reformat != formatted
+      return false if @formatted_fastas.all? { |ff| ff.v4? || ff.alias? }
+      return false if @formatted_fastas.all? { |ff| ff.v5? || ff.alias? }
+      true
     end
 
     # Runs makeblastdb on each file in `@fastas_to_format` and
@@ -109,8 +92,8 @@ module SequenceServer
       # Make the intent clear as well as ensure the program won't crash if
       # we accidentally call reformat before calling scan.
       return unless @fastas_to_reformat
-      @fastas_to_reformat.select do |path, title, type|
-        make_blast_database('reformat', path, title, type)
+      @fastas_to_reformat.select do |path, title, type, non_parse_seqids|
+        make_blast_database('reformat', path, title, type, non_parse_seqids)
       end
     end
 
@@ -120,34 +103,36 @@ module SequenceServer
     # formatted. Adds to @formatted_fastas.
     def determine_formatted_fastas
       blastdbcmd.each_line do |line|
-        path, title, type, *rest = line.split('	')
+        path, *rest = line.chomp.split("\t")
         next if multipart_database_name?(path)
-        @formatted_fastas << [path, title, type.strip.downcase, *rest]
+        rest << get_categories(path)
+        @formatted_fastas << Database.new(path, *rest)
       end
     end
 
     # Determines which FASTA files in the database directory require
     # reformatting. Adds to @fastas_to_format.
     def determine_fastas_to_reformat
-      @formatted_fastas.each do |path, title, type|
-        required_extensions = REQUIRED_EXTENSIONS[type]
-        exts = Dir["#{path}.*"].map { |p| p.split('.').last }.sort
-        next if (exts & required_extensions) == required_extensions
-
-        @fastas_to_reformat << [path, title, type]
+      @formatted_fastas.each do |ff|
+        if ff.v4? || ff.non_parse_seqids?
+          @fastas_to_reformat << [ff.path, ff.title, ff.type, ff.non_parse_seqids?]
+        end
       end
     end
 
     # Determines which FASTA files in the database directory are
     # unformatted. Adds to @fastas_to_format.
     def determine_unformatted_fastas
-      Find.find(database_dir) do |path|
+      # Add a trailing slash to database_dir - Find.find doesn't work as
+      # expected without the trailing slash if database_dir is a symlink
+      # inside a docker container.
+      Find.find(database_dir + '/') do |path|
         next if File.directory?(path)
         next unless probably_fasta?(path)
         next if @formatted_fastas.any? { |f| f[0] == path }
 
         @fastas_to_format << [path,
-          make_db_title(File.basename(path)),
+          make_db_title(path),
           guess_sequence_type_in_fasta(path)]
       end
     end
@@ -157,7 +142,7 @@ module SequenceServer
     # by `determine_formatted_fastas`.
     def blastdbcmd
       cmd = "blastdbcmd -recursive -list #{config[:database_dir]}" \
-            ' -list_outfmt "%f	%t	%p	%n	%l	%d"'
+            ' -list_outfmt "%f	%t	%p	%n	%l	%d	%v"'
       out, err = sys(cmd, path: config[:bin])
       errpat = /BLAST Database error/
       fail BLAST_DATABASE_ERROR.new(cmd, err) if err.match(errpat)
@@ -167,10 +152,11 @@ module SequenceServer
     end
 
     # Create BLAST database, given FASTA file and sequence type in FASTA file.
-    def make_blast_database(action, file, title, type)
+    def make_blast_database(action, file, title, type, non_parse_seqids = false)
       return unless make_blast_database?(action, file, type)
       title = confirm_database_title(title)
-      taxonomy = taxid_map(file) || taxid
+      extract_fasta(file) unless File.exist?(file)
+      taxonomy = taxid_map(file, non_parse_seqids) || taxid
       _make_blast_database(file, type, title, taxonomy)
     end
 
@@ -200,11 +186,10 @@ module SequenceServer
 
     # Check if a '.taxid_map.txt' file exists. If not, try getting it
     # using blastdbcmd.
-    def taxid_map(db)
+    def taxid_map(db, non_parse_seqids)
+      return if non_parse_seqids
       taxid_map = db.sub(/#{File.extname(db)}$/, '.taxid_map.txt')
-      if !File.exist?(taxid_map) || File.zero?(taxid_map)
-        extract_taxid_map(db, taxid_map)
-      end
+      extract_taxid_map(db, taxid_map) if !File.exist?(taxid_map)
       "-taxid_map #{taxid_map}" if !File.zero?(taxid_map)
     end
 
@@ -213,22 +198,23 @@ module SequenceServer
     # Using 0 as taxid is equivalent to not setting taxid for the database
     # that will be created.
     def taxid
+      default = 0
       print 'Enter taxid (optional): '
       user_response = STDIN.gets.strip
-      "-taxid #{user_response ? Integer(user_response) : 0}"
+      "-taxid #{user_response.empty? && default || Integer(user_response)}"
     rescue ArgumentError # presumably from call to Interger()
       puts 'taxid should be a number'
       retry
     end
 
     def _make_blast_database(file, type, title, taxonomy)
-      extract_fasta(file) unless File.exist?(file)
-      cmd = "makeblastdb -parse_seqids -hash_index -in #{file} " \
-            "-dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
+      cmd = "makeblastdb -parse_seqids -hash_index -in '#{file}'" \
+            " -dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
             " #{taxonomy}"
       out, err = sys(cmd, path: config[:bin])
       puts out.strip
       puts err.strip
+      return true
     rescue CommandFailed => e
       puts <<~MSG
         Could not create BLAST database for: #{file}
@@ -278,8 +264,16 @@ module SequenceServer
       !(db_name.match(%r{.+/\S+\.\d{2,3}$}).nil?)
     end
 
+    def get_categories(path)
+      path
+        .gsub(config[:database_dir], '') # remove database_dir from path
+        .split('/')
+        .reject(&:empty?)[0..-2] # the first entry might be '' if database_dir does not end with /
+    end
+
     # Returns true if first character of the file is '>'.
     def probably_fasta?(file)
+      return false unless file.match(/((cds)|(fasta)|(fna)|(pep)|(cdna)|(fa)|(prot)|(fas)|(genome)|(nuc)|(dna)|(nt))$/i)
       File.read(file, 1) == '>'
     end
 
@@ -288,7 +282,8 @@ module SequenceServer
     # For example:
     # Cobs1.4.proteins.fasta -> Cobs 1.4 proteins
     # S_invicta.xx.2.5.small.nucl.fa -> S invicta xx 2.5 small nucl
-    def make_db_title(db_name)
+    def make_db_title(path)
+      db_name = File.basename(path)
       db_name.tr!('"', "'")
       # removes .fasta like extension names
       db_name.gsub!(File.extname(db_name), '')

data/lib/sequenceserver/refinements.rb

@@ -0,0 +1,15 @@
+class Hash
+  def deep_copy
+    Marshal.load Marshal.dump self
+  end
+
+  def deep_merge(other)
+    merge(other) do |key, oldval, newval|
+      if oldval.is_a? Hash then
+        oldval.deep_merge(newval)
+      else
+        newval
+      end
+    end
+  end
+end

data/lib/sequenceserver/routes.rb

@@ -1,6 +1,7 @@
 require 'json'
 require 'tilt/erb'
 require 'sinatra/base'
+require 'rest-client'
 
 require 'sequenceserver/job'
 require 'sequenceserver/blast'
@@ -79,6 +80,8 @@ module SequenceServer
         options: SequenceServer.config[:options]
       }
 
+      searchdata.update(tree: Database.tree) if SequenceServer.config[:databases_widget] == 'tree'
+
       # If a job_id is specified, update searchdata from job meta data (i.e.,
       # query, pre-selected databases, advanced options used). Query is only
       # updated if params[:query] is not specified.
@@ -134,7 +137,7 @@ module SequenceServer
       database_ids = params['database_ids'].split(',')
       sequences = Sequence::Retriever.new(sequence_ids, database_ids, true)
       send_file(sequences.file.path,
-                type:     sequences.mime,
+                type: sequences.mime,
                 filename: sequences.filename)
     end
 
@@ -145,6 +148,57 @@ module SequenceServer
       send_file out.file, filename: out.filename, type: out.mime
     end
 
+    post '/cloud_share' do
+      content_type :json
+      request_params = JSON.parse(request.body.read)
+      job = Job.fetch(request_params['job_id'])
+
+      unless job.done?
+        status 422
+        { errors: ["Job #{request_params['job_id']} is not finished yet."] }.to_json
+      end
+
+      unless SequenceServer.config[:cloud_share_url]
+        status 503
+        { errors: ['Sorry, cloud sharing is not enabled on this server.'] }.to_json
+      end
+
+      begin
+        job.as_archived_file do |archived_job_file|
+          cloud_share_response = RestClient.post(
+            SequenceServer.config[:cloud_share_url],
+            {
+              shared_job: {
+                sender: {
+                  email: request_params['sender_email']
+                },
+                archived_job_file: archived_job_file,
+                original_job_id: job.id
+              }
+            }
+          )
+
+          return cloud_share_response.body
+        end
+      rescue RestClient::ExceptionWithResponse => e
+        cloud_share_response = e.response
+
+        case cloud_share_response.code
+        when 413
+          halt 413,
+               { errors: ['Sorry, the results are too large to share, please consider \
+                  using https://sequenceserver.com/cloud'] }.to_json
+        when 422
+          halt 422, JSON.parse(cloud_share_response.body).to_json
+        else
+          error cloud_share_response.code,
+                { errors: ["Unexpected Cloudshare response: #{cloud_share_response.code}"] }.to_json
+        end
+      rescue Errno::ECONNREFUSED
+        error 503, { errors: ['Sorry, the cloud sharing server may not be running. Try again later.'] }.to_json
+      end
+    end
+
     # Catches any exception raised within the app and returns JSON
     # representation of the error:
     # {
@@ -169,6 +223,7 @@ module SequenceServer
     # more_info.
     error 400..500 do
       error = env['sinatra.error']
+      return unless error
 
       # All errors will have a message.
       error_data = { message: error.message }
@@ -189,7 +244,13 @@ module SequenceServer
         error_data[:more_info] = error.backtrace.join("\n")
       end
 
-      error_data.to_json
+      if request.env['HTTP_ACCEPT'].to_s.include?('application/json')
+        content_type :json
+        error_data.to_json
+      else
+        content_type :html
+        erb :error, locals: { error_data: error_data }, layout: true
+      end
     end
 
     # Get the query sequences, selected databases, and advanced params used.
@@ -198,7 +259,7 @@ module SequenceServer
       return if job.imported_xml_file
 
       # Only read job.qfile if we are not going to use Database.retrieve.
-      searchdata[:query] = File.read(job.qfile) if !params[:query]
+      searchdata[:query] = File.read(job.qfile) unless params[:query]
 
       # Which databases to pre-select.
       searchdata[:preSelectedDbs] = job.databases
@@ -209,7 +270,12 @@ module SequenceServer
       # default advanced parameters from the advanced params input field. In
       # this case, we do want the advanced params input field to be empty when
       # the user hits the back button. Thus we do not test for empty string.
-      searchdata[:options][job.method] = [job.advanced] if job.advanced
+      method = job.method.to_sym
+      if job.advanced && job.advanced !=
+                         searchdata[:options][method][:default].join(' ')
+        searchdata[:options] = searchdata[:options].deep_copy
+        searchdata[:options][method]['last search'] = [job.advanced]
+      end
     end
   end
 end

data/lib/sequenceserver/sequence.rb

@@ -136,6 +136,7 @@ module SequenceServer
         # Returns a file name to use for the temporary file.
         def filename
           return @filename if @filename
+
           name = sequence_ids.first            if sequence_ids.length == 1
           name = "#{sequence_ids.length}_hits" if sequence_ids.length >= 2
           @filename = "sequenceserver-#{name}.fa"
@@ -182,14 +183,12 @@ module SequenceServer
         validate && run
       end
 
-      attr_reader :sequence_ids, :database_ids, :in_file
-
-      attr_reader :sequences
+      attr_reader :sequence_ids, :database_ids, :in_file, :sequences
 
-      def to_json
+      def to_json(*_args)
         {
           error_msgs: error_msgs,
-          sequences:  sequences.map(&:info)
+          sequences: sequences.map(&:info)
         }.to_json
       end
 
@@ -224,13 +223,15 @@ module SequenceServer
         ids = Database.ids
         return true if database_ids.is_a?(Array) && !database_ids.empty? &&
                        (ids & database_ids).length == database_ids.length
-        fail ArgumentError, 'Database id should be one of:' \
-                            " #{ids.join("\n")}."
+
+        fail DatabaseUnreachableError, 'Database id should be one of:' \
+                            " #{ids.join("\n")}"
       end
 
       # rubocop:disable Metrics/MethodLength
       def error_msgs
         return [] if sequences.length == sequence_ids.length
+
         [
           ['ERROR: incorrect number of sequences found.',
            <<~MSG

data/lib/sequenceserver/sys.rb

@@ -47,8 +47,8 @@ module SequenceServer
 
       # Execute the shell command, redirect stdout and stderr to the
       # temporary files.
-      exec("#{command} 1>#{temp_files[:stdout].path}" \
-           " 2>#{temp_files[:stderr].path}")
+      exec(command, out: temp_files[:stdout].path.to_s, \
+                    err: temp_files[:stderr].path.to_s)
     end
 
     # Wait for the termination of the child process.

data/lib/sequenceserver/version.rb

@@ -1,4 +1,4 @@
 # Define version number.
 module SequenceServer
-  VERSION = '2.0.0.rc8'.freeze
+  VERSION = '2.1.0'.freeze
 end