sequenceserver 2.2.0 → 3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 61b15f95cdc065739de70f2ba94d3022e6beec789dd9c64199ee1be3a2459c63
-  data.tar.gz: d6c108bd39cb8c6a832787d2cbaa52b2c5a3653c0b08b289cb149b7221a9870c
+  metadata.gz: 68dcb3dda53edae5d095423c164568484e786b927fcf733a4722eafde7c3a155
+  data.tar.gz: afb985fe5e762b8a1ab9e1dc4e8dd0bc6158f601a3cc95c0029f03a8587c1fa2
 SHA512:
-  metadata.gz: d4c2b1785980dd4e46a24a8a5a6d36652fb23f5389f9b7320cd9360f081cbad82b6dc6bfe735b5321738611094052631af0de635b77676587d703d466bd5f604
-  data.tar.gz: 235e35e8ba4331894a7ac9f107d950462d3b3d892ddafc6f58d3ba48fbe6a03cfc016ac64798921472e296a0cb5577245aca49cbb517d897094a501ec4483beb
+  metadata.gz: fb122a17858e6b7c567a418bda05358ae1a9b34ef1dfdcc11bb5f83097b47e427d7344e61e9f9102488f59f6448592b7d66e1e67ee213498e750dfea5e2e7c8f
+  data.tar.gz: b7b7504faf439acda18aafa0866e74f3645e431b4252bdb12ffd4896d7cf6ee065756db9e1bba027184794f306ce3b732c4911dce56d8b4a96e734dcf86c336c

data/COPYRIGHT.txt

@@ -3,7 +3,7 @@ SequenceServer is copyright Anurag Priyam, Ben J Woodcroft and Yannick Wurm,
 version 3, the text of which can be found in LICENSE.txt.
 
 Components of SequenceServer, including Sinatra, Ox, Slop, html5shiv,
-Underscore, jQuery, jQuery UI, Bootstrap and Webshim, are licensed under
+Underscore, jQuery, jQuery UI and Bootstrap, are licensed under
 the MIT license. D3.js is licensed under BSD license. biojs-vis-sequence
 is licensed under Apache license. Thin and JSON are licensed under Ruby
 license. All unmodified files from these and other sources retain their

data/bin/sequenceserver

@@ -358,7 +358,9 @@ begin
       end
 
       if make_blast_databases?
-        if SequenceServer.makeblastdb.any_to_format_or_reformat?
+        if SequenceServer.makeblastdb.no_fastas?
+          puts "Couldn't find any FASTA files in #{SequenceServer.config[:database_dir]}."
+        elsif SequenceServer.makeblastdb.any_to_format_or_reformat?
           puts
           puts <<~MSG
           SequenceServer has scanned your databases directory and will now offer
@@ -380,7 +382,7 @@ begin
           print '>> '
           response = STDIN.gets.to_s.strip
           SequenceServer.makeblastdb.run unless response =~ /^[n]$/i
-          else
+        else
           puts "All FASTA files in #{SequenceServer.config[:database_dir]} " \
                'are formatted.'
         end

data/lib/sequenceserver/blast/error.rb

@@ -0,0 +1,53 @@
+# http://www.ncbi.nlm.nih.gov/books/NBK1763/ (Appendices)
+
+module SequenceServer
+  module BLAST
+    class Error
+      attr_reader :exitstatus, :stdout, :stderr
+
+      def initialize(exitstatus:, stdout:, stderr:)
+        @exitstatus = exitstatus
+        @stdout = stdout
+        @stderr = stderr
+      end
+
+      def raise!
+        return true if exitstatus.zero? && !File.zero?(stdout)
+
+        case exitstatus
+        when 1..2
+          # 1: Error in query sequences or options.
+          # 2: Error in BLAST databases.
+          error = IO.foreach(stderr).grep(ERROR_LINE).join
+          error = File.read(stderr) if error.empty?
+          fail InputError, "(#{exitstatus}) #{error}"
+        when 4
+          # Out of memory. User can retry with a shorter search, so raising
+          # InputError here instead of SystemError.
+          fail InputError, <<~MSG
+            Ran out of memory. Please try a smaller query, fewer and smaller
+            databases, or limiting the output by using advanced options.
+          MSG
+        when 6
+          # Error creating output files. It can't be a permission issue as that
+          # would have been caught while creating job directory. But we can run
+          # out of storage after creating the job directory and while running
+          # the job. This is a SystemError.
+          fail SystemError, 'Ran out of disk space.'
+        else
+          # I am not sure what the exit codes 3 means and we should not
+          # encounter exit code 5. The only other error that I know can happen
+          # but is not yet handled is when BLAST+ binaries break such as after
+          # macOS updates. So raise SystemError, include the exit status in the
+          # message, and say that that the "most likely" reason is broken BLAST+
+          # binaries.
+
+          error = File.read(stderr)
+          error = 'Most likely there is a problem with the BLAST+ binaries.' if error.empty?
+
+          fail SystemError, "BLAST failed abruptly (exit status: #{exitstatus}). #{error}"
+        end
+      end
+    end
+  end
+end

data/lib/sequenceserver/blast/job.rb

@@ -1,5 +1,6 @@
 require 'sequenceserver/job'
 require 'sequenceserver/zip_file_generator'
+require 'sequenceserver/blast/error'
 
 module SequenceServer
   # BLAST module.
@@ -56,46 +57,8 @@ module SequenceServer
                      " -query '#{qfile}' #{options}"
       end
 
-      # Override Job#raise! to raise specific API errors based on exitstatus
-      # and using contents of stderr to provide context about the error.
-      #
       def raise!
-        # Return true exit status is 0 and stdout is not empty.
-        return true if exitstatus.zero? && !File.zero?(stdout)
-
-        # Handle error. See [1].
-        case exitstatus
-        when 1..2
-          # 1: Error in query sequences or options.
-          # 2: Error in BLAST databases.
-          error = IO.foreach(stderr).grep(ERROR_LINE).join
-          error = File.read(stderr) if error.empty?
-          fail InputError, "(#{exitstatus}) #{error}"
-        when 4
-          # Out of memory. User can retry with a shorter search, so raising
-          # InputError here instead of SystemError.
-          fail InputError, <<~MSG
-            Ran out of memory. Please try a smaller query, fewer and smaller
-            databases, or limiting the output by using advanced options.
-          MSG
-        when 6
-          # Error creating output files. It can't be a permission issue as that
-          # would have been caught while creating job directory. But we can run
-          # out of storage after creating the job directory and while running
-          # the job. This is a SystemError.
-          fail SystemError, 'Ran out of disk space.'
-        else
-          # I am not sure what the exit codes 3 means and we should not
-          # encounter exit code 5. The only other error that I know can happen
-          # but is not yet handled is when BLAST+ binaries break such as after
-          # macOS updates. So raise SystemError, include the exit status in the
-          # message, and say that that the "most likely" reason is broken BLAST+
-          # binaries.
-          fail SystemError, <<~MSG
-            BLAST failed abruptly (exit status: #{exitstatus}). Most likely there is a
-            problem with the BLAST+ binaries.
-          MSG
-        end
+        SequenceServer::BLAST::Error.new(exitstatus: exitstatus, stdout: stdout, stderr: stderr).raise!
       end
 
       # Use it with a block to get a self-cleaning temporary archive file
@@ -189,7 +152,3 @@ module SequenceServer
     end
   end
 end
-
-# References
-# ----------
-# [1]: http://www.ncbi.nlm.nih.gov/books/NBK1763/ (Appendices)

data/lib/sequenceserver/job.rb

@@ -28,16 +28,25 @@ module SequenceServer
         enqueue(job)
       end
 
+      def serializable_classes
+        [
+          Time,
+          Symbol,
+          SequenceServer::Job,
+          SequenceServer::BLAST::Job,
+          SequenceServer::Database
+        ]
+      end
+
       # Fetches job with the given id.
       def fetch(id)
         job_file = File.join(DOTDIR, id, 'job.yaml')
-        fail NotFound unless File.exist?(job_file)
-        if RUBY_VERSION < '3.1.0'
-          YAML.load_file(job_file)
-        else
-          YAML.load_file(job_file, permitted_classes: [Time, Symbol, SequenceServer::BLAST::Job, \
-                                                       SequenceServer::Database, SequenceServer::Job])
-        end
+        return nil unless File.exist?(job_file)
+
+        YAML.safe_load_file(
+          job_file,
+          permitted_classes: serializable_classes
+        )
       end
 
       # Deletes job with the given id.
@@ -75,8 +84,8 @@ module SequenceServer
     # of job data will be held, yields (if block given) and saves the job.
     #
     # Subclasses should extend `initialize` as per requirement.
-    def initialize(*)
-      @id = SecureRandom.uuid
+    def initialize(params = {})
+      @id = params.fetch(:id, SecureRandom.uuid)
       @submitted_at = Time.now
       mkdir_p dir
       yield if block_given?
@@ -85,7 +94,7 @@ module SequenceServer
       raise SystemError, 'Not enough disk space to start a new job'
     rescue Errno::EACCES
       raise SystemError, "Permission denied to write to #{DOTDIR}"
-    rescue => e
+    rescue StandardError => e
       rm_rf dir
       raise e
     end
@@ -117,7 +126,7 @@ module SequenceServer
     # should be called on a completed job before attempting to use the results.
     # Subclasses should provide their own implementation.
     def raise!
-      raise if done? && exitstatus != 0
+      fail if done? && exitstatus != 0
     end
 
     # Where will the stdout be written to during execution and read from later.
@@ -163,6 +172,7 @@ module SequenceServer
     def fetch(key)
       filename = File.join(dir, key)
       fail unless File.exist? filename
+
       filename
     end
 

data/lib/sequenceserver/makeblastdb-modified-with-cache.rb

@@ -0,0 +1,345 @@
+require 'find'
+require 'forwardable'
+
+module SequenceServer
+  # Smart `makeblastdb` wrapper: recursively scans database directory determining
+  # which files need to be formatted or re-formatted.
+  #
+  # Example usage:
+  #
+  #   makeblastdb = MAKEBLASTDB.new(database_dir)
+  #   makeblastdb.scan && makeblastdb.run
+  #
+  class MAKEBLASTDB
+    extend Forwardable
+
+    def_delegators SequenceServer, :config, :sys, :logger
+
+    def initialize(database_dir)
+      @database_dir = database_dir
+    end
+
+    attr_reader :database_dir
+    attr_reader :formatted_fastas
+    attr_reader :fastas_to_format
+    attr_reader :fastas_to_reformat
+
+    # Scans the database directory to determine which FASTA files require
+    # formatting or re-formatting.
+    #
+    # Returns `true` if there are files to (re-)format, `false` otherwise.
+    def scan
+      # We need to know the list of formatted FASTAs as reported by blastdbcmd
+      # first. This is required to determine both unformatted FASTAs and those
+      # that require reformatting.
+      @formatted_fastas = []
+      determine_formatted_fastas
+
+      # Now determine FASTA files that are unformatted or require reformatting.
+      @fastas_to_format = []
+      determine_unformatted_fastas
+      @fastas_to_reformat = []
+      determine_fastas_to_reformat
+
+      # Return true if there are files to be (re-)formatted or false otherwise.
+      !@fastas_to_format.empty? || !@fastas_to_reformat.empty?
+    end
+
+    # Returns true if at least one database in database directory is formatted.
+    def any_formatted?
+      !@formatted_fastas.empty?
+    end
+
+    # Returns true if there is at least one unformatted FASTA in the databases
+    # directory.
+    def any_unformatted?
+      !@fastas_to_format.empty?
+    end
+
+    # Returns true if the databases directory contains one or more incompatible
+    # databases.
+    #
+    # Note that it is okay to only use V4 databases or only V5 databases.
+    # Incompatibility arises when they are mixed.
+    def any_incompatible?
+      return false if @formatted_fastas.all? { |ff| ff.v4? || ff.alias? }
+      return false if @formatted_fastas.all? { |ff| ff.v5? || ff.alias? }
+      true
+    end
+
+    # Runs makeblastdb on each file in `@fastas_to_format` and
+    # `@fastas_to_reformat`. Will do nothing unless `#scan`
+    # has been run before.
+    def run
+      format
+      reformat
+    end
+
+    # Format any unformatted FASTA files in database directory. Returns Array
+    # of files that were formatted.
+    def format
+      # Make the intent clear as well as ensure the program won't crash if we
+      # accidentally call format before calling scan.
+      return unless @fastas_to_format
+      @fastas_to_format.select do |path, title, type|
+        make_blast_database('format', path, title, type)
+      end
+    end
+
+    # Re-format databases that require reformatting. Returns Array of files
+    # that were reformatted.
+    def reformat
+      # Make the intent clear as well as ensure the program won't crash if
+      # we accidentally call reformat before calling scan.
+      return unless @fastas_to_reformat
+      @fastas_to_reformat.select do |path, title, type, non_parse_seqids|
+        make_blast_database('reformat', path, title, type, non_parse_seqids)
+      end
+    end
+
+    private
+
+    # Determines which FASTA files in the database directory are already
+    # formatted. Adds to @formatted_fastas.
+    def determine_formatted_fastas
+      blastdbcmd.each_line do |line|
+        path, *rest = line.chomp.split("\t")
+        next if multipart_database_name?(path)
+        rest << get_categories(path)
+        @formatted_fastas << Database.new(path, *rest)
+      end
+    end
+
+    # Determines which FASTA files in the database directory require
+    # reformatting. Adds to @fastas_to_format.
+    def determine_fastas_to_reformat
+      @formatted_fastas.each do |ff|
+        if ff.v4? || ff.non_parse_seqids?
+          @fastas_to_reformat << [ff.path, ff.title, ff.type, ff.non_parse_seqids?]
+        end
+      end
+    end
+
+    # Determines which FASTA files in the database directory are
+    # unformatted. Adds to @fastas_to_format.
+    def determine_unformatted_fastas
+      # Add a trailing slash to database_dir - Find.find doesn't work as
+      # expected without the trailing slash if database_dir is a symlink
+      # inside a docker container.
+      Find.find(database_dir + '/') do |path|
+        next if File.directory?(path)
+        next unless probably_fasta?(path)
+        next if @formatted_fastas.any? { |f| f[0] == path }
+
+        @fastas_to_format << [path,
+          make_db_title(path),
+          guess_sequence_type_in_fasta(path)]
+      end
+    end
+
+    # Runs `blastdbcmd` to determine formatted FASTA files in the database
+    # directory. Returns the output of `blastdbcmd`. This method is called
+    # by `determine_formatted_fastas`.
+    def blastdbcmd
+      # calculate checksum of database directory
+      current_db_checksum = Zlib::crc32(Dir.glob(File.join(config[:database_dir], '/**/*')).map {
+        |path| path.to_s + "_" + File.mtime(path).to_s + "_" + File.size(path).to_s
+      }.to_s)
+
+      checksum_path = config[:database_dir].chomp('/') + '.checksum'
+      index_path    = config[:database_dir].chomp('/') + '.index'
+
+      if File.exists?(checksum_path)
+        if current_db_checksum == File.read(checksum_path).to_i # db directory hasn't changed
+          if File.exists?(index_path) # lets use existing index
+            logger.info "Using existing database index: #{index_path}"
+            return File.read(index_path)
+          end
+        end
+      end rescue logger.error "Could not read: #{checksum_path} or #{index_path}"
+
+      # database directory has changed, or index file doesn't exist
+      # thus we run blastdbcmd to get formatted FASTA files
+      logger.info "Scanning for BLAST databases & creating index"
+      cmd = "blastdbcmd -recursive -list #{config[:database_dir]}" \
+            ' -list_outfmt "%f	%t	%p	%n	%l	%d	%v"'
+      out, err = sys(cmd, path: config[:bin])
+      errpat = /BLAST Database error/
+      fail BLAST_DATABASE_ERROR.new(cmd, err) if err.match(errpat)
+
+      # write checksum and index to file
+      File.open(checksum_path, 'w') { |f| f.write(current_db_checksum) } rescue
+         logger.error "Could not write database checksum to file" + checksum_path
+      File.open(index_path, 'w') { |f| f.write(out) } rescue
+         logger.error "Could not write database index to file" + index_path
+
+      return out
+    rescue CommandFailed => e
+      fail BLAST_DATABASE_ERROR.new(cmd, e.stderr)
+    end
+
+    # Create BLAST database, given FASTA file and sequence type in FASTA file.
+    def make_blast_database(action, file, title, type, non_parse_seqids = false)
+      return unless make_blast_database?(action, file, type)
+      title = confirm_database_title(title)
+      extract_fasta(file) unless File.exist?(file)
+      taxonomy = taxid_map(file, non_parse_seqids) || taxid
+      _make_blast_database(file, type, title, taxonomy)
+    end
+
+    # Show file path and guessed sequence type to the user and obtain a y/n
+    # response.
+    #
+    # Returns true if the user entered anything but 'n' or 'N'.
+    def make_blast_database?(action, file, type)
+      puts
+      puts
+      puts "FASTA file to #{action}: #{file}"
+      puts "FASTA type: #{type}"
+      print 'Proceed? [y/n] (Default: y): '
+      response = STDIN.gets.to_s.strip
+      !response.match(/n/i)
+    end
+
+    # Show the database title that we are going to use to the user for
+    # confirmation.
+    #
+    # Returns user input if any. Auto-determined title otherwise.
+    def confirm_database_title(default)
+      print "Enter a database title or will use '#{default}': "
+      from_user = STDIN.gets.to_s.strip
+      from_user.empty? && default || from_user
+    end
+
+    # Check if a '.taxid_map.txt' file exists. If not, try getting it
+    # using blastdbcmd.
+    def taxid_map(db, non_parse_seqids)
+      return if non_parse_seqids
+      taxid_map = db.sub(/#{File.extname(db)}$/, '.taxid_map.txt')
+      extract_taxid_map(db, taxid_map) if !File.exist?(taxid_map)
+      "-taxid_map #{taxid_map}" if !File.zero?(taxid_map)
+    end
+
+    # Get taxid from the user. Returns user input or 0.
+    #
+    # Using 0 as taxid is equivalent to not setting taxid for the database
+    # that will be created.
+    def taxid
+      default = 0
+      print 'Enter taxid (optional): '
+      user_response = STDIN.gets.strip
+      "-taxid #{user_response.empty? && default || Integer(user_response)}"
+    rescue ArgumentError # presumably from call to Interger()
+      puts 'taxid should be a number'
+      retry
+    end
+
+    def _make_blast_database(file, type, title, taxonomy)
+      cmd = "makeblastdb -parse_seqids -hash_index -in '#{file}'" \
+            " -dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
+            " #{taxonomy}"
+      out, err = sys(cmd, path: config[:bin])
+      puts out.strip
+      puts err.strip
+      return true
+    rescue CommandFailed => e
+      puts <<~MSG
+        Could not create BLAST database for: #{file}
+        Tried: #{cmd}
+        stdout: #{e.stdout}
+        stderr: #{e.stderr}
+      MSG
+      exit!
+    end
+
+    # Extract FASTA file from BLAST database.
+    #
+    # Invoked while reformatting a BLAST database if the corresponding
+    # FASTA file does not exist.
+    def extract_fasta(db)
+      puts
+      puts 'Extracting sequences ...'
+      cmd = "blastdbcmd -entry all -db #{db}"
+      sys(cmd, stdout: db, path: config[:bin])
+    rescue CommandFailed => e
+      puts <<~MSG
+        Could not extract sequences from: #{db}
+        Tried: #{cmd}
+        stdout: #{e.stdout}
+        stderr: #{e.stderr}
+      MSG
+      exit!
+    end
+
+    def extract_taxid_map(db, taxmap_file)
+      cmd = "blastdbcmd -entry all -db #{db} -outfmt '%i %T'"
+      sys(cmd, stdout: taxmap_file, path: config[:bin])
+    rescue CommandFailed => e
+      # silence error
+    end
+
+    # Returns true if the database name appears to be a multi-part database
+    # name.
+    #
+    # e.g.
+    # /home/ben/pd.ben/sequenceserver/db/nr.00 => yes
+    # /home/ben/pd.ben/sequenceserver/db/nr => no
+    # /home/ben/pd.ben/sequenceserver/db/img3.5.finished.faa.01 => yes
+    # /home/ben/pd.ben/sequenceserver/db/nr00 => no
+    # /mnt/blast-db/refseq_genomic.100 => yes
+    def multipart_database_name?(db_name)
+      !(db_name.match(%r{.+/\S+\.\d{2,3}$}).nil?)
+    end
+
+    def get_categories(path)
+      path
+        .gsub(config[:database_dir], '') # remove database_dir from path
+        .split('/')
+        .reject(&:empty?)[0..-2] # the first entry might be '' if database_dir does not end with /
+    end
+
+    # Returns true if first character of the file is '>'.
+    def probably_fasta?(file)
+      return false unless file.match(/((cds)|(fasta)|(fna)|(pep)|(cdna)|(fa)|(prot)|(fas)|(genome)|(nuc)|(dna)|(nt))$/i)
+      File.read(file, 1) == '>'
+    end
+
+    # Suggests improved titles when generating database names from files
+    # for improved apperance and readability in web interface.
+    # For example:
+    # Cobs1.4.proteins.fasta -> Cobs 1.4 proteins
+    # S_invicta.xx.2.5.small.nucl.fa -> S invicta xx 2.5 small nucl
+    def make_db_title(path)
+      db_name = File.basename(path)
+      db_name.tr!('"', "'")
+      # removes .fasta like extension names
+      db_name.gsub!(File.extname(db_name), '')
+      # replaces _ with ' ',
+      db_name.gsub!(/(_)/, ' ')
+      # replaces '.' with ' ' when no numbers are on either side,
+      db_name.gsub!(/(\D)\.(?=\D)/, '\1 ')
+      # preserves version numbers
+      db_name.gsub!(/\W*(\d+([.-]\d+)+)\W*/, ' \1 ')
+      db_name
+    end
+
+    # Guess whether FASTA file contains protein or nucleotide sequences by
+    # sampling a few few characters of the file.
+    def guess_sequence_type_in_fasta(file)
+      sequences = sample_sequences(file)
+      sequence_types = sequences.map { |seq| Sequence.guess_type(seq) }
+      sequence_types = sequence_types.uniq.compact
+      (sequence_types.length == 1) && sequence_types.first
+    end
+
+    # Read first 1,048,576 characters of the file, split the read text on
+    # fasta def line pattern and return.
+    #
+    # If the given file is FASTA, returns Array of as many different
+    # sequences in the portion of the file read. Returns the portion
+    # of the file read wrapped in an Array otherwise.
+    def sample_sequences(file)
+      File.read(file, 1_048_576).split(/^>.+$/).delete_if(&:empty?)
+    end
+  end
+end

data/lib/sequenceserver/makeblastdb.rb

@@ -30,6 +30,10 @@ module SequenceServer
       any_to_format? || any_to_reformat?
     end
 
+    def no_fastas?
+      probably_fastas.empty?
+    end
+
     # Runs makeblastdb on each file in `fastas_to_format` and
     # `fastas_to_reformat`.
     def run
@@ -79,12 +83,12 @@ module SequenceServer
       @formatted_fastas
     end
 
-    private
-
     def any_to_format?
       fastas_to_format.any?
     end
 
+    private
+
     def any_to_reformat?
       fastas_to_reformat.any?
     end
@@ -107,22 +111,32 @@ module SequenceServer
     def fastas_to_format
       return @fastas_to_format if defined?(@fastas_to_format)
 
-      @fastas_to_format = []
+      formatted_fasta_paths = formatted_fastas.map { |f| f[0] }
+      fasta_paths_to_format = probably_fastas - formatted_fasta_paths
+
+      @fastas_to_format = fasta_paths_to_format.map do |path|
+        [
+          path,
+          make_db_title(path),
+          guess_sequence_type_in_fasta(path)
+        ]
+      end
+
+      @fastas_to_format
+    end
+
+    def probably_fastas
+      return @probably_fastas if defined?(@probably_fastas)
+
+      @probably_fastas = []
 
-      # Add a trailing slash to database_dir - Find.find doesn't work as
-      # expected without the trailing slash if database_dir is a symlink
-      # inside a docker container.
       Find.find(database_dir + '/') do |path|
         next if File.directory?(path)
-        next unless probably_fasta?(path)
-        next if formatted_fastas.any? { |f| f[0] == path }
 
-        @fastas_to_format << [path,
-                              make_db_title(path),
-                              guess_sequence_type_in_fasta(path)]
+        @probably_fastas << path if probably_fasta?(path)
       end
 
-      @fastas_to_format
+      @probably_fastas
     end
 
     # Runs `blastdbcmd` to determine formatted FASTA files in the database