sequenceserver 2.0.0.rc4 → 2.0.0.rc5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56e26f480b73ad075d19001178316bce0bf7a5b26eba33a36dfb17db8ad8e525
-  data.tar.gz: 9b9e8f88a1941181a8e71e89d93e07f4b2a47d07cfb44e6691333e56eb31537a
+  metadata.gz: b04c2f43509bf7d4a15d555e3bd9cea86d3c7770c7a3e35016f963f877055ecf
+  data.tar.gz: 25678397eacb2c3f445902624baff80a90d2a47ca98d2b703e13de3eca84cf1e
 SHA512:
-  metadata.gz: 07b11c9ed4f89c81d49d73a0a10fde2a89326dab10cefc8794ed13bc165831d234393cf36cdc58c1552ce06bde2057e468e6e17fbaa438c4a31d0036c65591b5
-  data.tar.gz: 406996da9f95d6c9c2463448d9f4ff72dd33fb70d73bf2e1bbb926c17c641d37d89e9f9c18156ac31d9fc50d5141338ed70e522fc46d8e35903dde9b015e79f6
+  metadata.gz: a8e6a4c1b4917d9bd35d07ef8e580e1b75105bfd1709fb892d9ed523da91f8c3a08d50c1c9da304530cd0bb5d77da00a4944c65cd5ce90fc979c9d6099149058
+  data.tar.gz: ef493ec4cf1aef63b2f535251143b34154d0d332a47a1d15668c7a74b139d5e6c2a2f9c33b5c7072ae9c2fbbcf33048fad4e828273c85a199a6ab79874d6e766

data/bin/sequenceserver

@@ -68,10 +68,6 @@ begin
         # of threads to use in config file.
         $ sequenceserver -s -n 16
 
-        # See if you have FASTA files in database dir that haven't
-        # been converted into BLAST database.
-        $ sequenceserver -u
-
         # Search for FASTA files in database dir that haven't been
         # converted into BLAST database yet, and convert them.
         $ sequenceserver -m
@@ -135,9 +131,6 @@ begin
     on 'l', 'list_databases',
        'List BLAST databases'
 
-    on 'u', 'list-unformatted-fastas',
-       'List unformatted FASTA files'
-
     on 'i', 'interactive',
        'Run SequenceServer in interactive mode'
 
@@ -285,8 +278,7 @@ begin
         fetch_option(:database_dir).value = response
         redo
       rescue SequenceServer::NO_BLAST_DATABASE_FOUND => e
-        unless list_databases? || list_unformatted_fastas? ||
-               make_blast_databases?
+        unless list_databases? || make_blast_databases?
 
           # Print error raised.
           puts
@@ -305,13 +297,13 @@ begin
           unless response =~ /^[n]$/i
             puts
             puts 'Searching ...'
-            if SequenceServer::Database.unformatted_fastas.empty?
-              puts "Couldn't find any FASTA files."
-              exit!
-            else
-              formatted = SequenceServer::Database.make_blast_databases
+            if SequenceServer.makeblastdb.scan
+              formatted = SequenceServer.makeblastdb.run
               exit! if formatted.empty? && !set?
               redo unless set?
+            else
+              puts "Couldn't find any FASTA files."
+              exit!
             end
           else
             exit! unless set?
@@ -361,22 +353,13 @@ begin
         exit
       end
 
-      if list_unformatted_fastas? || make_blast_databases?
-        unformatted_fastas = SequenceServer::Database.unformatted_fastas
-        if unformatted_fastas.empty?
+      if make_blast_databases?
+        if SequenceServer.makeblastdb.scan
+          SequenceServer.makeblastdb.run
+        else
           puts "All FASTA files in #{SequenceServer.config[:database_dir]} " \
                'are formatted.'
-          exit
         end
-      end
-
-      if list_unformatted_fastas?
-        puts unformatted_fastas
-        exit
-      end
-
-      if make_blast_databases?
-        SequenceServer::Database.make_blast_databases
         exit
       end
 

data/lib/sequenceserver.rb

@@ -24,6 +24,7 @@ module SequenceServer
   require 'sequenceserver/config'
   require 'sequenceserver/server'
   require 'sequenceserver/routes'
+  require 'sequenceserver/makeblastdb'
   require 'sequenceserver/job_remover'
   require 'sequenceserver/exceptions'
   require 'sequenceserver/sys'
@@ -57,6 +58,11 @@ module SequenceServer
                   end
     end
 
+    # MAKEBLASTDB service object.
+    def makeblastdb
+      @makeblastdb ||= MAKEBLASTDB.new(config[:database_dir])
+    end
+
     # SequenceServer initialisation routine.
     def init(config = {})
       # Use default config file if caller didn't specify one.

data/lib/sequenceserver/blast/job.rb

@@ -65,6 +65,16 @@ module SequenceServer
           error = IO.foreach(stderr).grep(ERROR_LINE).join
           error = File.read(stderr) if error.empty?
           fail InputError, error
+        when 2
+          fail InputError, <<~MSG
+            BLAST signalled a problem with the databases that you searched.
+
+            Most likely one or more of your databases were created using an
+            older version of BLAST. Please consider recreating the databases
+            using BLAST #{BLAST_VERSION}.
+
+            As a temporary solution, you can try searching one database at a time.
+          MSG
         when 4
           # Out of memory. User can retry with a shorter search, so raising
           # InputError here instead of SystemError.
@@ -79,7 +89,7 @@ module SequenceServer
           # the job. This is a SystemError.
           fail SystemError, 'Ran out of disk space.'
         else
-          # I am not sure what the exit codes 2 & 3 means and we should note
+          # I am not sure what the exit codes 3 means and we should not
           # encounter exit code 5. The only other error that I know can happen
           # but is not yet handled is when BLAST+ binaries break such as after
           # macOS updates. So raise SystemError, include the exit status in the

data/lib/sequenceserver/database.rb

@@ -1,4 +1,3 @@
-require 'find'
 require 'open3'
 require 'digest/md5'
 require 'forwardable'
@@ -209,97 +208,6 @@ module SequenceServer
       end
       # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
-      # Recursively scan `database_dir` for un-formatted FASTA and format them
-      # for use with BLAST+.
-      def make_blast_databases
-        unformatted_fastas.select do |file, sequence_type|
-          make_blast_database(file, sequence_type)
-        end
-      end
-
-      # Returns an Array of FASTA files that may require formatting, and the
-      # type of sequence contained in each FASTA.
-      #
-      #   > unformatted_fastas
-      #   => [['/foo/bar.fasta', :nulceotide], ...]
-      def unformatted_fastas
-        list = []
-        database_dir = config[:database_dir]
-        Find.find database_dir do |file|
-          next if File.directory? file
-          next if Database.include? file
-          next unless probably_fasta? file
-          sequence_type = guess_sequence_type_in_fasta file
-          if %i[protein nucleotide].include?(sequence_type)
-            list << [file, sequence_type]
-          end
-        end
-        list
-      end
-
-      # Create BLAST database, given FASTA file and sequence type in FASTA file.
-      def make_blast_database(file, type)
-        return unless make_blast_database? file, type
-        title = get_database_title(file)
-        taxid = fetch_tax_id
-        _make_blast_database(file, type, title, taxid)
-      end
-
-      def _make_blast_database(file, type, title, taxid, quiet = false)
-        cmd = 'makeblastdb -parse_seqids -hash_index ' \
-              "-in #{file} -dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
-              " -taxid #{taxid}"
-        out, err = sys(cmd, path: config[:bin])
-        puts out, err unless quiet
-      rescue CommandFailed => e
-        puts <<~MSG
-          Could not create BLAST database for: #{file}
-          Tried: #{cmd}
-          stdout: #{e.stdout}
-          stderr: #{e.stderr}
-        MSG
-        exit!
-      end
-
-      # Show file path and guessed sequence type to the user and obtain a y/n
-      # response.
-      #
-      # Returns true if the user entered anything but 'n' or 'N'.
-      def make_blast_database?(file, type)
-        puts
-        puts
-        puts "FASTA file: #{file}"
-        puts "FASTA type: #{type}"
-        print 'Proceed? [y/n] (Default: y): '
-        response = STDIN.gets.to_s.strip
-        !response.match(/n/i)
-      end
-
-      # Generate a title for the given database and show it to the user for
-      # confirmation.
-      #
-      # Returns user input if any. Auto-generated title otherwise.
-      def get_database_title(path)
-        default = make_db_title(File.basename(path))
-        print "Enter a database title or will use '#{default}': "
-        from_user = STDIN.gets.to_s.strip
-        from_user.empty? && default || from_user
-      end
-
-      # Get taxid from the user. Returns user input or 0.
-      #
-      # Using 0 as taxid is equivalent to not setting taxid for the database
-      # that will be created.
-      def fetch_tax_id
-        default = 0
-        print 'Enter taxid (optional): '
-        user_response = STDIN.gets.strip
-        user_response.empty? && default || Integer(user_response)
-      rescue
-        puts 'taxid should be a number'
-        retry
-      end
-
       # Returns true if the database name appears to be a multi-part database
       # name.
       #
@@ -312,48 +220,6 @@ module SequenceServer
       def multipart_database_name?(db_name)
         !(db_name.match(%r{.+/\S+\.\d{2,3}$}).nil?)
       end
-
-      # Returns true if first character of the file is '>'.
-      def probably_fasta?(file)
-        File.read(file, 1) == '>'
-      end
-
-      # Suggests improved titles when generating database names from files
-      # for improved apperance and readability in web interface.
-      # For example:
-      # Cobs1.4.proteins.fasta -> Cobs 1.4 proteins
-      # S_invicta.xx.2.5.small.nucl.fa -> S invicta xx 2.5 small nucl
-      def make_db_title(db_name)
-        db_name.tr!('"', "'")
-        # removes .fasta like extension names
-        db_name.gsub!(File.extname(db_name), '')
-        # replaces _ with ' ',
-        db_name.gsub!(/(_)/, ' ')
-        # replaces '.' with ' ' when no numbers are on either side,
-        db_name.gsub!(/(\D)\.(?=\D)/, '\1 ')
-        # preserves version numbers
-        db_name.gsub!(/\W*(\d+([.-]\d+)+)\W*/, ' \1 ')
-        db_name
-      end
-
-      # Guess whether FASTA file contains protein or nucleotide sequences by
-      # sampling a few few characters of the file.
-      def guess_sequence_type_in_fasta(file)
-        sequences = sample_sequences(file)
-        sequence_types = sequences.map { |seq| Sequence.guess_type(seq) }
-        sequence_types = sequence_types.uniq.compact
-        (sequence_types.length == 1) && sequence_types.first
-      end
-
-      # Read first 1,048,576 characters of the file, split the read text on
-      # fasta def line pattern and return.
-      #
-      # If the given file is FASTA, returns Array of as many different
-      # sequences in the portion of the file read. Returns the portion
-      # of the file read wrapped in an Array otherwise.
-      def sample_sequences(file)
-        File.read(file, 1_048_576).split(/^>.+$/).delete_if(&:empty?)
-      end
     end
   end
 end

data/lib/sequenceserver/makeblastdb.rb

@@ -0,0 +1,243 @@
+require 'find'
+require 'forwardable'
+
+module SequenceServer
+  # Smart `makeblastdb` wrapper: recursively scans database directory determining
+  # which files need to be formatted or re-formatted.
+  #
+  # Example usage:
+  #
+  #   makeblastdb = MAKEBLASTDB.new(database_dir)
+  #   makeblastdb.scan && makeblastdb.run
+  #
+  class MAKEBLASTDB
+    # We want V5 databases created using -parse_seqids for proper function of
+    # SequenceServer. This means each database should be comprised of at least 9
+    # files with the following extensions. Databases created by us will have two
+    # additional files with the extensions nhd and nhi, or phd and phi, due to
+    # the use of -hash_index option. Finally, multipart databases will have one
+    # additional file with the extension nal or pal.
+    REQUIRED_EXTENSIONS = {
+      'nucleotide' => %w{ndb nhr nin nog nos not nsq ntf nto}.freeze,
+      'protein' => %w{pdb phr pin pog pos pot psq ptf pto}.freeze
+    }
+
+    extend Forwardable
+
+    def_delegators SequenceServer, :config, :sys
+
+    def initialize(database_dir)
+      @database_dir = database_dir
+    end
+
+    attr_reader :database_dir
+
+    # Scans the database directory to determine which FASTA files require
+    # formatting or re-formatting.
+    #
+    # Returns `true` if there are files to (re-)format, `false` otherwise.
+    def scan
+      # We need to know the list of formatted FASTAs as reported by blastdbcmd
+      # first. This is required to determine both unformatted FASTAs and those
+      # that require reformatting.
+      @formatted_fastas = []
+      determine_formatted_fastas
+
+      # Now determine FASTA files that are unformatted or require reformatting.
+      @fastas_to_format = []
+      determine_unformatted_fastas
+      determine_fastas_to_reformat
+
+      # Return true if there are files to be (re-)formatted or false otherwise.
+      !@fastas_to_format.empty?
+    end
+
+    # Runs makeblastdb on each file in `@fastas_to_format`. Will do nothing
+    # unless `#scan` has been run before.
+    def run
+      return unless @fastas_to_format || @fastas_to_format.empty?
+      @fastas_to_format.each do |path, title, type|
+        make_blast_database(path, title, type)
+      end
+    end
+
+    private
+
+    # Determines which FASTA files in the database directory are already
+    # formatted. Adds to @formatted_fastas.
+    def determine_formatted_fastas
+      blastdbcmd.each_line do |line|
+        path, title, type = line.split('	')
+        next if multipart_database_name?(path)
+        @formatted_fastas << [path, title, type.strip.downcase]
+      end
+    end
+
+    # Determines which FASTA files in the database directory require
+    # reformatting. Adds to @fastas_to_format.
+    def determine_fastas_to_reformat
+      @formatted_fastas.each do |path, title, type|
+        required_extensions = REQUIRED_EXTENSIONS[type]
+        exts = Dir["#{path}.*"].map { |p| p.split('.').last }.sort
+        next if (exts & required_extensions) == required_extensions
+
+        @fastas_to_format << [path, title, type]
+      end
+    end
+
+    # Determines which FASTA files in the database directory are
+    # unformatted. Adds to @fastas_to_format.
+    def determine_unformatted_fastas
+      Find.find(database_dir) do |path|
+        next if File.directory?(path)
+        next unless probably_fasta?(path)
+        next if @formatted_fastas.any? { |f| f[0] == path }
+
+        @fastas_to_format << [path,
+          make_db_title(File.basename(path)),
+          guess_sequence_type_in_fasta(path)]
+      end
+    end
+
+    # Runs `blastdbcmd` to determine formatted FASTA files in the database
+    # directory. Returns the output of `blastdbcmd`. This method is called
+    # by `determine_formatted_fastas`.
+    def blastdbcmd
+      cmd = "blastdbcmd -recursive -list #{database_dir}" \
+            ' -list_outfmt "%f	%t	%p"'
+      out, _ = sys(cmd, path: config[:bin])
+      out
+    end
+
+    # Create BLAST database, given FASTA file and sequence type in FASTA file.
+    def make_blast_database(file, title, type)
+      return unless make_blast_database? file, type
+      title = confirm_database_title(title)
+      taxid = fetch_tax_id
+      _make_blast_database(file, type, title, taxid)
+    end
+
+    def _make_blast_database(file, type, title, taxid)
+      extract_fasta(file) unless File.exist?(file)
+      cmd = "makeblastdb -parse_seqids -hash_index -in #{file} " \
+            "-dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
+            " -taxid #{taxid}"
+      out, err = sys(cmd, path: config[:bin])
+      puts out.strip
+      puts err.strip
+    rescue CommandFailed => e
+      puts <<~MSG
+        Could not create BLAST database for: #{file}
+        Tried: #{cmd}
+        stdout: #{e.stdout}
+        stderr: #{e.stderr}
+      MSG
+      exit!
+    end
+
+    # Show file path and guessed sequence type to the user and obtain a y/n
+    # response.
+    #
+    # Returns true if the user entered anything but 'n' or 'N'.
+    def make_blast_database?(file, type)
+      puts
+      puts
+      puts "FASTA file to format/reformat: #{file}"
+      puts "FASTA type: #{type}"
+      print 'Proceed? [y/n] (Default: y): '
+      response = STDIN.gets.to_s.strip
+      !response.match(/n/i)
+    end
+
+    # Show the database title that we are going to use to the user for
+    # confirmation.
+    #
+    # Returns user input if any. Auto-determined title otherwise.
+    def confirm_database_title(default)
+      print "Enter a database title or will use '#{default}': "
+      from_user = STDIN.gets.to_s.strip
+      from_user.empty? && default || from_user
+    end
+
+    # Get taxid from the user. Returns user input or 0.
+    #
+    # Using 0 as taxid is equivalent to not setting taxid for the database
+    # that will be created.
+    def fetch_tax_id
+      default = 0
+      print 'Enter taxid (optional): '
+      user_response = STDIN.gets.strip
+      user_response.empty? && default || Integer(user_response)
+    rescue
+      puts 'taxid should be a number'
+      retry
+    end
+
+    # Extract FASTA file from BLAST database.
+    #
+    # Invoked while reformatting a BLAST database if the corresponding
+    # FASTA file does not exist.
+    def extract_fasta(db)
+      puts
+      puts 'Extracting sequences ...'
+      cmd = "blastdbcmd -entry all -db #{db}"
+      sys(cmd, stdout: db, path: config[:bin])
+    rescue CommandFailed => e
+      puts <<~MSG
+        Could not extract sequences from: #{db}
+        Tried: #{cmd}
+        stdout: #{e.stdout}
+        stderr: #{e.stderr}
+      MSG
+      exit!
+    end
+
+    # Returns true if the database name appears to be a multi-part database
+    # name.
+    def multipart_database_name?(db_name)
+      Database.multipart_database_name? db_name
+    end
+
+    # Returns true if first character of the file is '>'.
+    def probably_fasta?(file)
+      File.read(file, 1) == '>'
+    end
+
+    # Suggests improved titles when generating database names from files
+    # for improved apperance and readability in web interface.
+    # For example:
+    # Cobs1.4.proteins.fasta -> Cobs 1.4 proteins
+    # S_invicta.xx.2.5.small.nucl.fa -> S invicta xx 2.5 small nucl
+    def make_db_title(db_name)
+      db_name.tr!('"', "'")
+      # removes .fasta like extension names
+      db_name.gsub!(File.extname(db_name), '')
+      # replaces _ with ' ',
+      db_name.gsub!(/(_)/, ' ')
+      # replaces '.' with ' ' when no numbers are on either side,
+      db_name.gsub!(/(\D)\.(?=\D)/, '\1 ')
+      # preserves version numbers
+      db_name.gsub!(/\W*(\d+([.-]\d+)+)\W*/, ' \1 ')
+      db_name
+    end
+
+    # Guess whether FASTA file contains protein or nucleotide sequences by
+    # sampling a few few characters of the file.
+    def guess_sequence_type_in_fasta(file)
+      sequences = sample_sequences(file)
+      sequence_types = sequences.map { |seq| Sequence.guess_type(seq) }
+      sequence_types = sequence_types.uniq.compact
+      (sequence_types.length == 1) && sequence_types.first
+    end
+
+    # Read first 1,048,576 characters of the file, split the read text on
+    # fasta def line pattern and return.
+    #
+    # If the given file is FASTA, returns Array of as many different
+    # sequences in the portion of the file read. Returns the portion
+    # of the file read wrapped in an Array otherwise.
+    def sample_sequences(file)
+      File.read(file, 1_048_576).split(/^>.+$/).delete_if(&:empty?)
+    end
+  end
+end