sequenceserver 2.0.0 → 2.2.0

data/lib/sequenceserver/makeblastdb.rb

@@ -8,7 +8,8 @@ module SequenceServer
   # Example usage:
   #
   #   makeblastdb = MAKEBLASTDB.new(database_dir)
-  #   makeblastdb.scan && makeblastdb.run
+  #   makeblastdb.run # formats and re-formats databases in database_dir
+  #   makeblastdb.formatted_fastas # lists formatted databases
   #
   class MAKEBLASTDB
     extend Forwardable
@@ -20,56 +21,17 @@ module SequenceServer
     end
 
     attr_reader :database_dir
-    attr_reader :formatted_fastas
-    attr_reader :fastas_to_format
-    attr_reader :fastas_to_reformat
 
-    # Scans the database directory to determine which FASTA files require
-    # formatting or re-formatting.
-    #
-    # Returns `true` if there are files to (re-)format, `false` otherwise.
-    def scan
-      # We need to know the list of formatted FASTAs as reported by blastdbcmd
-      # first. This is required to determine both unformatted FASTAs and those
-      # that require reformatting.
-      @formatted_fastas = []
-      determine_formatted_fastas
-
-      # Now determine FASTA files that are unformatted or require reformatting.
-      @fastas_to_format = []
-      determine_unformatted_fastas
-      @fastas_to_reformat = []
-      determine_fastas_to_reformat
-
-      # Return true if there are files to be (re-)formatted or false otherwise.
-      !@fastas_to_format.empty? || !@fastas_to_reformat.empty?
-    end
-
-    # Returns true if at least one database in database directory is formatted.
     def any_formatted?
-      !@formatted_fastas.empty?
-    end
-
-    # Returns true if there is at least one unformatted FASTA in the databases
-    # directory.
-    def any_unformatted?
-      !@fastas_to_format.empty?
+      formatted_fastas.any?
     end
 
-    # Returns true if the databases directory contains one or more incompatible
-    # databases.
-    #
-    # Note that it is okay to only use V4 databases or only V5 databases.
-    # Incompatibility arises when they are mixed.
-    def any_incompatible?
-      return false if @formatted_fastas.all? { |ff| ff.v4? || ff.alias? }
-      return false if @formatted_fastas.all? { |ff| ff.v5? || ff.alias? }
-      true
+    def any_to_format_or_reformat?
+      any_to_format? || any_to_reformat?
     end
 
-    # Runs makeblastdb on each file in `@fastas_to_format` and
-    # `@fastas_to_reformat`. Will do nothing unless `#scan`
-    # has been run before.
+    # Runs makeblastdb on each file in `fastas_to_format` and
+    # `fastas_to_reformat`.
     def run
       format
       reformat
@@ -80,8 +42,9 @@ module SequenceServer
     def format
       # Make the intent clear as well as ensure the program won't crash if we
       # accidentally call format before calling scan.
-      return unless @fastas_to_format
-      @fastas_to_format.select do |path, title, type|
+      return unless any_to_format?
+
+      fastas_to_format.select do |path, title, type|
         make_blast_database('format', path, title, type)
       end
     end
@@ -91,50 +54,75 @@ module SequenceServer
     def reformat
       # Make the intent clear as well as ensure the program won't crash if
       # we accidentally call reformat before calling scan.
-      return unless @fastas_to_reformat
-      @fastas_to_reformat.select do |path, title, type, non_parse_seqids|
+      return unless any_to_reformat?
+
+      fastas_to_reformat.select do |path, title, type, non_parse_seqids|
         make_blast_database('reformat', path, title, type, non_parse_seqids)
       end
     end
 
-    private
-
     # Determines which FASTA files in the database directory are already
-    # formatted. Adds to @formatted_fastas.
-    def determine_formatted_fastas
+    # formatted.
+    def formatted_fastas
+      return @formatted_fastas if defined?(@formatted_fastas)
+
+      @formatted_fastas = []
+
       blastdbcmd.each_line do |line|
         path, *rest = line.chomp.split("\t")
         next if multipart_database_name?(path)
+
         rest << get_categories(path)
         @formatted_fastas << Database.new(path, *rest)
       end
+
+      @formatted_fastas
+    end
+
+    private
+
+    def any_to_format?
+      fastas_to_format.any?
+    end
+
+    def any_to_reformat?
+      fastas_to_reformat.any?
     end
 
     # Determines which FASTA files in the database directory require
-    # reformatting. Adds to @fastas_to_format.
-    def determine_fastas_to_reformat
-      @formatted_fastas.each do |ff|
-        if ff.v4? || ff.non_parse_seqids?
-          @fastas_to_reformat << [ff.path, ff.title, ff.type, ff.non_parse_seqids?]
-        end
+    # reformatting.
+    def fastas_to_reformat
+      return @fastas_to_reformat if defined?(@fastas_to_reformat)
+
+      @fastas_to_reformat = []
+      formatted_fastas.each do |ff|
+        @fastas_to_reformat << [ff.path, ff.title, ff.type, ff.non_parse_seqids?] if ff.v4? || ff.non_parse_seqids?
       end
+
+      @fastas_to_reformat
     end
 
     # Determines which FASTA files in the database directory are
-    # unformatted. Adds to @fastas_to_format.
-    def determine_unformatted_fastas
+    # unformatted.
+    def fastas_to_format
+      return @fastas_to_format if defined?(@fastas_to_format)
+
+      @fastas_to_format = []
+
       # Add a trailing slash to database_dir - Find.find doesn't work as
       # expected without the trailing slash if database_dir is a symlink
       # inside a docker container.
       Find.find(database_dir + '/') do |path|
         next if File.directory?(path)
         next unless probably_fasta?(path)
-        next if @formatted_fastas.any? { |f| f[0] == path }
+        next if formatted_fastas.any? { |f| f[0] == path }
 
         @fastas_to_format << [path,
-          make_db_title(path),
-          guess_sequence_type_in_fasta(path)]
+                              make_db_title(path),
+                              guess_sequence_type_in_fasta(path)]
       end
+
+      @fastas_to_format
     end
 
     # Runs `blastdbcmd` to determine formatted FASTA files in the database
@@ -146,14 +134,16 @@ module SequenceServer
       out, err = sys(cmd, path: config[:bin])
       errpat = /BLAST Database error/
       fail BLAST_DATABASE_ERROR.new(cmd, err) if err.match(errpat)
-      return out
+
+      out
     rescue CommandFailed => e
-      fail BLAST_DATABASE_ERROR.new(cmd, e.stderr)
+      raise BLAST_DATABASE_ERROR.new(cmd, e.stderr)
     end
 
     # Create BLAST database, given FASTA file and sequence type in FASTA file.
     def make_blast_database(action, file, title, type, non_parse_seqids = false)
       return unless make_blast_database?(action, file, type)
+
       title = confirm_database_title(title)
       extract_fasta(file) unless File.exist?(file)
       taxonomy = taxid_map(file, non_parse_seqids) || taxid
@@ -188,9 +178,10 @@ module SequenceServer
     # using blastdbcmd.
     def taxid_map(db, non_parse_seqids)
       return if non_parse_seqids
+
       taxid_map = db.sub(/#{File.extname(db)}$/, '.taxid_map.txt')
-      extract_taxid_map(db, taxid_map) if !File.exist?(taxid_map)
-      "-taxid_map #{taxid_map}" if !File.zero?(taxid_map)
+      extract_taxid_map(db, taxid_map) unless File.exist?(taxid_map)
+      "-taxid_map #{taxid_map}" unless File.zero?(taxid_map)
     end
 
     # Get taxid from the user. Returns user input or 0.
@@ -211,10 +202,24 @@ module SequenceServer
       cmd = "makeblastdb -parse_seqids -hash_index -in '#{file}'" \
             " -dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
             " #{taxonomy}"
-      out, err = sys(cmd, path: config[:bin])
+
+      output = if File.directory?(file)
+                 File.join(file, 'makeblastdb')
+               else
+                 "#{file}.makeblastdb"
+               end
+
+      out, err = sys(
+        cmd,
+        path: config[:bin],
+        stderr: [output, 'stderr'].join,
+        stdout: [output, 'stdout'].join
+      )
+
       puts out.strip
       puts err.strip
-      return true
+
+      true
     rescue CommandFailed => e
       puts <<~MSG
         Could not create BLAST database for: #{file}
@@ -261,7 +266,7 @@ module SequenceServer
     # /home/ben/pd.ben/sequenceserver/db/nr00 => no
     # /mnt/blast-db/refseq_genomic.100 => yes
     def multipart_database_name?(db_name)
-      !(db_name.match(%r{.+/\S+\.\d{2,3}$}).nil?)
+      !db_name.match(%r{.+/\S+\.\d{2,3}$}).nil?
     end
 
     def get_categories(path)
@@ -273,7 +278,10 @@ module SequenceServer
 
     # Returns true if first character of the file is '>'.
     def probably_fasta?(file)
-      return false unless file.match(/((cds)|(fasta)|(fna)|(pep)|(cdna)|(fa)|(prot)|(fas)|(genome)|(nuc)|(dna)|(nt))$/i)
+      unless file.match(/((cdna)|(cds)|(dna)|(fa)|(faa)|(fas)|(fasta)|(fna)|(genome)|(nt)|(nuc)|(pep)|(prot))$/i)
+        return false
+      end
+
       File.read(file, 1) == '>'
     end
 

data/lib/sequenceserver/pool.rb

@@ -42,7 +42,7 @@ class Pool
 
   def shutdown
     @size.times do
-      schedule { throw :exit }
+      queue { throw :exit }
     end
     @pool.map(&:join)
   end

data/lib/sequenceserver/refinements.rb

@@ -0,0 +1,15 @@
+class Hash
+  def deep_copy
+    Marshal.load Marshal.dump self
+  end
+
+  def deep_merge(other)
+    merge(other) do |key, oldval, newval|
+      if oldval.is_a? Hash then
+        oldval.deep_merge(newval)
+      else
+        newval
+      end
+    end
+  end
+end

data/lib/sequenceserver/report.rb

@@ -7,11 +7,8 @@ module SequenceServer
   # own report subclass.
   class Report
     class << self
-      # Generates report for the given job. Returns generated report object.
-      #
-      # TODO: Dynamic dispatch.
       def generate(job)
-        BLAST::Report.new(job)
+        BLAST::Report.new(job).generate
       end
     end
 
@@ -23,7 +20,6 @@ module SequenceServer
     def initialize(job)
       @job = job
       yield if block_given?
-      generate
     end
 
     attr_reader :job

data/lib/sequenceserver/routes.rb

@@ -1,6 +1,7 @@
 require 'json'
 require 'tilt/erb'
 require 'sinatra/base'
+require 'rest-client'
 
 require 'sequenceserver/job'
 require 'sequenceserver/blast'
@@ -79,9 +80,7 @@ module SequenceServer
         options: SequenceServer.config[:options]
       }
 
-      if SequenceServer.config[:databases_widget] == 'tree'
-        searchdata.update(tree: Database.tree)
-      end
+      searchdata.update(tree: Database.tree) if SequenceServer.config[:databases_widget] == 'tree'
 
       # If a job_id is specified, update searchdata from job meta data (i.e.,
       # query, pre-selected databases, advanced options used). Query is only
@@ -107,7 +106,28 @@ module SequenceServer
     get '/:jid.json' do |jid|
       job = Job.fetch(jid)
       halt 202 unless job.done?
-      Report.generate(job).to_json
+
+      report = Report.generate(job)
+      halt 202 unless report.done?
+
+      display_large_result_warning =
+        SequenceServer.config[:large_result_warning_threshold].to_i.positive? &&
+        params[:bypass_file_size_warning] != 'true' &&
+        report.xml_file_size > SequenceServer.config[:large_result_warning_threshold]
+
+      if display_large_result_warning
+        halt 200,
+             {
+               user_warning: 'LARGE_RESULT',
+               download_links: [
+                 { name: 'Standard Tabular Report', url: "download/#{jid}.std_tsv" },
+                 { name: 'Full Tabular Report', url: "/download/#{jid}.full_tsv" },
+                 { name: 'Results in XML', url: "/download/#{jid}.xml" }
+               ]
+             }.to_json
+      end
+
+      report.to_json
     end
 
     # Returns base HTML. Rest happens client-side: polling for and rendering
@@ -138,7 +158,7 @@ module SequenceServer
       database_ids = params['database_ids'].split(',')
       sequences = Sequence::Retriever.new(sequence_ids, database_ids, true)
       send_file(sequences.file.path,
-                type:     sequences.mime,
+                type: sequences.mime,
                 filename: sequences.filename)
     end
 
@@ -146,7 +166,58 @@ module SequenceServer
     get '/download/:jid.:type' do |jid, type|
       job = Job.fetch(jid)
       out = BLAST::Formatter.new(job, type)
-      send_file out.file, filename: out.filename, type: out.mime
+      send_file out.filepath, filename: out.filename, type: out.mime
+    end
+
+    post '/cloud_share' do
+      content_type :json
+      request_params = JSON.parse(request.body.read)
+      job = Job.fetch(request_params['job_id'])
+
+      unless job.done?
+        status 422
+        { errors: ["Job #{request_params['job_id']} is not finished yet."] }.to_json
+      end
+
+      unless SequenceServer.config[:cloud_share_url]
+        status 503
+        { errors: ['Sorry, cloud sharing is not enabled on this server.'] }.to_json
+      end
+
+      begin
+        job.as_archived_file do |archived_job_file|
+          cloud_share_response = RestClient.post(
+            SequenceServer.config[:cloud_share_url],
+            {
+              shared_job: {
+                sender: {
+                  email: request_params['sender_email']
+                },
+                archived_job_file: archived_job_file,
+                original_job_id: job.id
+              }
+            }
+          )
+
+          return cloud_share_response.body
+        end
+      rescue RestClient::ExceptionWithResponse => e
+        cloud_share_response = e.response
+
+        case cloud_share_response.code
+        when 413
+          halt 413,
+               { errors: ['Sorry, the results are too large to share, please consider \
+                  using https://sequenceserver.com/cloud'] }.to_json
+        when 422
+          halt 422, JSON.parse(cloud_share_response.body).to_json
+        else
+          error cloud_share_response.code,
+                { errors: ["Unexpected Cloudshare response: #{cloud_share_response.code}"] }.to_json
+        end
+      rescue Errno::ECONNREFUSED
+        error 503, { errors: ['Sorry, the cloud sharing server may not be running. Try again later.'] }.to_json
+      end
     end
 
     # Catches any exception raised within the app and returns JSON
@@ -173,6 +244,7 @@ module SequenceServer
     # more_info.
     error 400..500 do
       error = env['sinatra.error']
+      return unless error
 
       # All errors will have a message.
       error_data = { message: error.message }
@@ -193,7 +265,13 @@ module SequenceServer
         error_data[:more_info] = error.backtrace.join("\n")
       end
 
-      error_data.to_json
+      if request.env['HTTP_ACCEPT'].to_s.include?('application/json')
+        content_type :json
+        error_data.to_json
+      else
+        content_type :html
+        erb :error, locals: { error_data: error_data }, layout: true
+      end
     end
 
     # Get the query sequences, selected databases, and advanced params used.
@@ -202,7 +280,7 @@ module SequenceServer
       return if job.imported_xml_file
 
       # Only read job.qfile if we are not going to use Database.retrieve.
-      searchdata[:query] = File.read(job.qfile) if !params[:query]
+      searchdata[:query] = File.read(job.qfile) unless params[:query]
 
       # Which databases to pre-select.
       searchdata[:preSelectedDbs] = job.databases
@@ -213,7 +291,12 @@ module SequenceServer
       # default advanced parameters from the advanced params input field. In
       # this case, we do want the advanced params input field to be empty when
       # the user hits the back button. Thus we do not test for empty string.
-      searchdata[:options][job.method] = [job.advanced] if job.advanced
+      method = job.method.to_sym
+      if job.advanced && job.advanced !=
+                         searchdata[:options][method][:default].join(' ')
+        searchdata[:options] = searchdata[:options].deep_copy
+        searchdata[:options][method]['last search'] = [job.advanced]
+      end
     end
   end
 end

data/lib/sequenceserver/sequence.rb

@@ -136,6 +136,7 @@ module SequenceServer
         # Returns a file name to use for the temporary file.
         def filename
           return @filename if @filename
+
           name = sequence_ids.first            if sequence_ids.length == 1
           name = "#{sequence_ids.length}_hits" if sequence_ids.length >= 2
           @filename = "sequenceserver-#{name}.fa"
@@ -182,14 +183,12 @@ module SequenceServer
         validate && run
       end
 
-      attr_reader :sequence_ids, :database_ids, :in_file
-
-      attr_reader :sequences
+      attr_reader :sequence_ids, :database_ids, :in_file, :sequences
 
-      def to_json
+      def to_json(*_args)
         {
           error_msgs: error_msgs,
-          sequences:  sequences.map(&:info)
+          sequences: sequences.map(&:info)
         }.to_json
       end
 
@@ -224,13 +223,15 @@ module SequenceServer
         ids = Database.ids
         return true if database_ids.is_a?(Array) && !database_ids.empty? &&
                        (ids & database_ids).length == database_ids.length
-        fail ArgumentError, 'Database id should be one of:' \
-                            " #{ids.join("\n")}."
+
+        fail DatabaseUnreachableError, 'Database id should be one of:' \
+                            " #{ids.join("\n")}"
       end
 
       # rubocop:disable Metrics/MethodLength
       def error_msgs
         return [] if sequences.length == sequence_ids.length
+
         [
           ['ERROR: incorrect number of sequences found.',
            <<~MSG

data/lib/sequenceserver/sys.rb

@@ -47,8 +47,8 @@ module SequenceServer
 
       # Execute the shell command, redirect stdout and stderr to the
       # temporary files.
-      exec("#{command} 1>#{temp_files[:stdout].path}" \
-           " 2>#{temp_files[:stderr].path}")
+      exec(command, out: temp_files[:stdout].path.to_s, \
+                    err: temp_files[:stderr].path.to_s)
     end
 
     # Wait for the termination of the child process.
@@ -67,7 +67,7 @@ module SequenceServer
 
       # Now move the temporary file to the given path.
       # TODO: don't we need to explicitly close the temp file here?
-      FileUtils.mv(temp_files.delete(channel), filename)
+      FileUtils.cp(temp_files[channel], filename)
     end
 
     # Read the remaining temp files into memory. For large outputs,

data/lib/sequenceserver/version.rb

@@ -1,4 +1,4 @@
 # Define version number.
 module SequenceServer
-  VERSION = '2.0.0'.freeze
+  VERSION = '2.2.0'.freeze
 end

data/lib/sequenceserver/zip_file_generator.rb

@@ -0,0 +1,56 @@
+# Source: https://github.com/rubyzip/rubyzip/blob/9d891f7353e66052283562d3e252fe380bb4b199/samples/example_recursive.rb
+
+require 'zip'
+
+# This is a simple example which uses rubyzip to
+# recursively generate a zip file from the contents of
+# a specified directory. The directory itself is not
+# included in the archive, rather just its contents.
+#
+# Usage:
+#   directory_to_zip = "/tmp/input"
+#   output_file = "/tmp/out.zip"
+#   zf = ZipFileGenerator.new(directory_to_zip, output_file)
+#   zf.write()
+class ZipFileGenerator
+  # Initialize with the directory to zip and the location of the output archive.
+  def initialize(input_dir, output_file)
+    @input_dir = input_dir
+    @output_file = output_file
+  end
+
+  # Zip the input directory.
+  def write
+    entries = Dir.entries(@input_dir) - %w[. ..]
+
+    ::Zip::File.open(@output_file, ::Zip::File::CREATE) do |zipfile|
+      write_entries entries, '', zipfile
+    end
+  end
+
+  private
+
+  # A helper method to make the recursion work.
+  def write_entries(entries, path, zipfile)
+    entries.each do |e|
+      zipfile_path = path == '' ? e : File.join(path, e)
+      disk_file_path = File.join(@input_dir, zipfile_path)
+
+      if File.directory? disk_file_path
+        recursively_deflate_directory(disk_file_path, zipfile, zipfile_path)
+      else
+        put_into_archive(disk_file_path, zipfile, zipfile_path)
+      end
+    end
+  end
+
+  def recursively_deflate_directory(disk_file_path, zipfile, zipfile_path)
+    zipfile.mkdir zipfile_path
+    subdir = Dir.entries(disk_file_path) - %w[. ..]
+    write_entries subdir, zipfile_path, zipfile
+  end
+
+  def put_into_archive(disk_file_path, zipfile, zipfile_path)
+    zipfile.add(zipfile_path, disk_file_path)
+  end
+end

data/lib/sequenceserver.rb

@@ -24,6 +24,7 @@ module SequenceServer
   require 'sequenceserver/job_remover'
   require 'sequenceserver/exceptions'
   require 'sequenceserver/sys'
+  require 'sequenceserver/refinements'
 
   # The singleton methods defined below constitute the "runtime" environment of
   # SequenceServer.
@@ -61,6 +62,9 @@ module SequenceServer
 
     # SequenceServer initialisation routine.
     def init(config = {})
+      # Reset makeblastdb cache, because configuration may have changed.
+      @makeblastdb = nil
+
       # Use default config file if caller didn't specify one.
       config[:config_file] ||= DEFAULT_CONFIG_FILE
 
@@ -200,10 +204,13 @@ module SequenceServer
 
       logger.debug("Will look for BLAST+ databases in: #{config[:database_dir]}")
 
-      makeblastdb.scan
-      fail NO_BLAST_DATABASE_FOUND, config[:database_dir] if !makeblastdb.any_formatted?
+      fail NO_BLAST_DATABASE_FOUND, config[:database_dir] unless makeblastdb.any_formatted?
 
       Database.collection = makeblastdb.formatted_fastas
+      check_database_compatibility unless config[:optimistic].to_s == 'true'
+    end
+
+    def check_database_compatibility
       Database.each do |database|
         logger.debug "Found #{database.type} database '#{database.title}' at '#{database.path}'"
         if database.non_parse_seqids?