sequenceserver 2.0.0.rc7 → 2.0.0.rc8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2f7be0b03d87a6322d843166c7cdcf1aea0236fdd82a932b9eeda936a43f4a53
-  data.tar.gz: eb13c9ab355e3424e979606273606c6334ee203b649126d9d2860cdbb7d76ee4
+  metadata.gz: 449efaa2619fd2c83a4116480c78ae6b892804f2f9bc23c3d074b8b0493985c5
+  data.tar.gz: 393f9da8bd3df53bff3ed3a137a5d9484ad5ec9cfa1de3c7b15c3f1a6833f092
 SHA512:
-  metadata.gz: 9183f80e0a6909e9e88b4f0416eeaac5ef50e4d75777764f82278b9377bbd5720f34c82cca9aff446eeb735ab79ef69872b8b6e8feabca161b8e81b52e995e4f
-  data.tar.gz: 99375aca038464b785883f12922656d5733d6cf114c37f62b83bc1c222f371b0b491ced709e678b2c07e52d8bae479ee8cdabdbaf06f1f91add2074f1e8652e5
+  metadata.gz: de2fdcef61a8fcdbe8907935a1f0b545219698fe9d37fcad6cd40aaa50e779010b7e91a9158eb76590baed78ce3743cdc173b664ff5d0969dfaca42eb6bd621c
+  data.tar.gz: a65c243d19435021b00a360ddfa8f617fa58e5944dd55117a8865f11dec4554a86e577cdfee3013a82638dc45de697a33156da1ba0a2fedd8e2936a7d760d5d9

data/bin/sequenceserver

@@ -40,7 +40,7 @@ def ask_to_join
       "-d ifq -d emailAddress=#{response} -d submit=Submit > /dev/null 2> /dev/null"
       system post_email_cmd
     end
-    system "touch #{asked_to_join}"
+    system "mkdir -p #{SequenceServer::DOTDIR} && touch #{asked_to_join}"
   end
 end
 
@@ -279,7 +279,6 @@ begin
         redo
       rescue SequenceServer::NO_BLAST_DATABASE_FOUND => e
         unless list_databases? || make_blast_databases?
-
           # Print error raised.
           puts
           puts e
@@ -297,8 +296,8 @@ begin
           unless response =~ /^[n]$/i
             puts
             puts 'Searching ...'
-            if SequenceServer.makeblastdb.scan
-              formatted = SequenceServer.makeblastdb.run
+            if SequenceServer.makeblastdb.any_unformatted?
+              formatted = SequenceServer.makeblastdb.format
               exit! if formatted.empty? && !set?
               redo unless set?
             else
@@ -309,6 +308,34 @@ begin
             exit! unless set?
           end
         end
+      rescue SequenceServer::INCOMPATIBLE_BLAST_DATABASES => e
+        unless list_databases? || make_blast_databases?
+          # Print error raised.
+          puts
+          puts e
+
+          # Offer user to format the FASTA files.
+          database_dir = SequenceServer.config[:database_dir]
+          puts
+          puts <<~MSG
+            Incompatible databases can cause BLAST searches and other features of
+            SequenceServer to fail unexpectedly.
+            You can view incompatible databases and choose to reformat them below.
+            Alternatively, please remove them from databases directory.
+
+            View incompatible databases? [y/n] (Default: y).
+          MSG
+          puts
+          print '>> '
+          response = STDIN.gets.to_s.strip
+          unless response =~ /^[n]$/i
+            reformatted = SequenceServer.makeblastdb.reformat
+            exit! if reformatted.empty? && !set?
+            redo unless set?
+          else
+            exit! unless set?
+          end
+        end
       rescue SequenceServer::ENOENT,
         SequenceServer::CONFIG_FILE_ERROR,
         SequenceServer::BLAST_DATABASE_ERROR,

data/lib/sequenceserver.rb

@@ -202,7 +202,12 @@ module SequenceServer
       end
 
       logger.debug("Will look for BLAST+ databases in: #{config[:database_dir]}")
-      Database.scan_databases_dir
+
+      makeblastdb.scan
+      fail NO_BLAST_DATABASE_FOUND, config[:database_dir] if !makeblastdb.any_formatted?
+      fail INCOMPATIBLE_BLAST_DATABASES, config[:database_dir] if makeblastdb.any_incompatible?
+
+      Database.collection = makeblastdb.formatted_fastas
       Database.each do |database|
         logger.debug("Found #{database.type} database '#{database.title}'" \
                      " at '#{database.name}'")

data/lib/sequenceserver/database.rb

@@ -80,12 +80,15 @@ module SequenceServer
         @collection ||= {}
       end
 
-      private :collection
-
-      def <<(database)
-        collection[database.id] = database
+      def collection=(databases_attrs)
+        databases_attrs.each do |db_attrs|
+          db = Database.new(*db_attrs)
+          collection[db.id] = db
+        end
       end
 
+      private :collection
+
       def [](ids)
         ids = Array ids
         collection.values_at(*ids)
@@ -182,44 +185,6 @@ module SequenceServer
       def clear
         collection.clear
       end
-
-      # Recurisvely scan `database_dir` for blast databases.
-      #
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      def blastdbcmd
-        cmd = "blastdbcmd -recursive -list #{config[:database_dir]}" \
-              ' -list_outfmt "%f	%t	%p	%n	%l	%d"'
-        out, err = sys(cmd, path: config[:bin])
-        errpat = /BLAST Database error/
-        fail BLAST_DATABASE_ERROR.new(cmd, err) if err.match(errpat)
-        return out
-      rescue CommandFailed => e
-        fail BLAST_DATABASE_ERROR.new(cmd, e.stderr)
-      end
-
-      def scan_databases_dir
-        out = blastdbcmd
-        fail NO_BLAST_DATABASE_FOUND, config[:database_dir] if out.empty?
-        out.each_line do |line|
-          name = line.split('	')[0]
-          next if multipart_database_name?(name)
-          self << Database.new(*line.split('	'))
-        end
-      end
-      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
-
-      # Returns true if the database name appears to be a multi-part database
-      # name.
-      #
-      # e.g.
-      # /home/ben/pd.ben/sequenceserver/db/nr.00 => yes
-      # /home/ben/pd.ben/sequenceserver/db/nr => no
-      # /home/ben/pd.ben/sequenceserver/db/img3.5.finished.faa.01 => yes
-      # /home/ben/pd.ben/sequenceserver/db/nr00 => no
-      # /mnt/blast-db/refseq_genomic.100 => yes
-      def multipart_database_name?(db_name)
-        !(db_name.match(%r{.+/\S+\.\d{2,3}$}).nil?)
-      end
     end
   end
 end

data/lib/sequenceserver/exceptions.rb

@@ -113,6 +113,20 @@ module SequenceServer
     end
   end
 
+  class INCOMPATIBLE_BLAST_DATABASES < StandardError
+    def initialize(database_dir)
+      @database_dir = database_dir
+    end
+
+    attr_reader :database_dir
+
+    def to_s
+      <<~MSG
+        One or more databases in #{database_dir} are likely incompatible.
+      MSG
+    end
+  end
+
   # Raised if there was an error determining BLAST+ databases in database_dir.
   class BLAST_DATABASE_ERROR < StandardError
     def initialize(cmd, out)

data/lib/sequenceserver/makeblastdb.rb

@@ -31,6 +31,9 @@ module SequenceServer
     end
 
     attr_reader :database_dir
+    attr_reader :formatted_fastas
+    attr_reader :fastas_to_format
+    attr_reader :fastas_to_reformat
 
     # Scans the database directory to determine which FASTA files require
     # formatting or re-formatting.
@@ -46,18 +49,68 @@ module SequenceServer
       # Now determine FASTA files that are unformatted or require reformatting.
       @fastas_to_format = []
       determine_unformatted_fastas
+      @fastas_to_reformat = []
       determine_fastas_to_reformat
 
       # Return true if there are files to be (re-)formatted or false otherwise.
+      !@fastas_to_format.empty? || !@fastas_to_reformat.empty?
+    end
+
+    # Returns true if at least one database in database directory is formatted.
+    def any_formatted?
+      !@formatted_fastas.empty?
+    end
+
+    # Returns true if there is at least one unformatted FASTA in the databases
+    # directory.
+    def any_unformatted?
       !@fastas_to_format.empty?
     end
 
-    # Runs makeblastdb on each file in `@fastas_to_format`. Will do nothing
-    # unless `#scan` has been run before.
+    # Returns true if the databases directory contains one or more incompatible
+    # databases.
+    #
+    # Note that it is okay to only use V4 databases or only V5 databases. It is
+    # Incompatibility arises when they are mixed.
+    def any_incompatible?
+      return false if @fastas_to_reformat.empty?
+      # We need to compare @fastas_to_reformat to @formatted_fastas, but the
+      # latter contains extra attributes. However, the first attribute, i.e,
+      # path is common to both and sufficient for our needs.
+      to_reformat = @fastas_to_reformat.map(&:first)
+      formatted = @formatted_fastas.map(&:first)
+      # Check that they are not equal. Using intersection operator ensures
+      # comparison even if their order differs.
+      formatted & to_reformat != formatted
+    end
+
+    # Runs makeblastdb on each file in `@fastas_to_format` and
+    # `@fastas_to_reformat`. Will do nothing unless `#scan`
+    # has been run before.
     def run
-      return unless @fastas_to_format || @fastas_to_format.empty?
-      @fastas_to_format.each do |path, title, type|
-        make_blast_database(path, title, type)
+      format
+      reformat
+    end
+
+    # Format any unformatted FASTA files in database directory. Returns Array
+    # of files that were formatted.
+    def format
+      # Make the intent clear as well as ensure the program won't crash if we
+      # accidentally call format before calling scan.
+      return unless @fastas_to_format
+      @fastas_to_format.select do |path, title, type|
+        make_blast_database('format', path, title, type)
+      end
+    end
+
+    # Re-format databases that require reformatting. Returns Array of files
+    # that were reformatted.
+    def reformat
+      # Make the intent clear as well as ensure the program won't crash if
+      # we accidentally call reformat before calling scan.
+      return unless @fastas_to_reformat
+      @fastas_to_reformat.select do |path, title, type|
+        make_blast_database('reformat', path, title, type)
       end
     end
 
@@ -67,9 +120,9 @@ module SequenceServer
     # formatted. Adds to @formatted_fastas.
     def determine_formatted_fastas
       blastdbcmd.each_line do |line|
-        path, title, type = line.split('	')
+        path, title, type, *rest = line.split('	')
         next if multipart_database_name?(path)
-        @formatted_fastas << [path, title, type.strip.downcase]
+        @formatted_fastas << [path, title, type.strip.downcase, *rest]
       end
     end
 
@@ -81,7 +134,7 @@ module SequenceServer
         exts = Dir["#{path}.*"].map { |p| p.split('.').last }.sort
         next if (exts & required_extensions) == required_extensions
 
-        @fastas_to_format << [path, title, type]
+        @fastas_to_reformat << [path, title, type]
       end
     end
 
@@ -103,46 +156,32 @@ module SequenceServer
     # directory. Returns the output of `blastdbcmd`. This method is called
     # by `determine_formatted_fastas`.
     def blastdbcmd
-      cmd = "blastdbcmd -recursive -list #{database_dir}" \
-            ' -list_outfmt "%f	%t	%p"'
-      out, _ = sys(cmd, path: config[:bin])
-      out
+      cmd = "blastdbcmd -recursive -list #{config[:database_dir]}" \
+            ' -list_outfmt "%f	%t	%p	%n	%l	%d"'
+      out, err = sys(cmd, path: config[:bin])
+      errpat = /BLAST Database error/
+      fail BLAST_DATABASE_ERROR.new(cmd, err) if err.match(errpat)
+      return out
+    rescue CommandFailed => e
+      fail BLAST_DATABASE_ERROR.new(cmd, e.stderr)
     end
 
     # Create BLAST database, given FASTA file and sequence type in FASTA file.
-    def make_blast_database(file, title, type)
-      return unless make_blast_database? file, type
+    def make_blast_database(action, file, title, type)
+      return unless make_blast_database?(action, file, type)
       title = confirm_database_title(title)
-      taxid = fetch_tax_id
-      _make_blast_database(file, type, title, taxid)
-    end
-
-    def _make_blast_database(file, type, title, taxid)
-      extract_fasta(file) unless File.exist?(file)
-      cmd = "makeblastdb -parse_seqids -hash_index -in #{file} " \
-            "-dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
-            " -taxid #{taxid}"
-      out, err = sys(cmd, path: config[:bin])
-      puts out.strip
-      puts err.strip
-    rescue CommandFailed => e
-      puts <<~MSG
-        Could not create BLAST database for: #{file}
-        Tried: #{cmd}
-        stdout: #{e.stdout}
-        stderr: #{e.stderr}
-      MSG
-      exit!
+      taxonomy = taxid_map(file) || taxid
+      _make_blast_database(file, type, title, taxonomy)
     end
 
     # Show file path and guessed sequence type to the user and obtain a y/n
     # response.
     #
     # Returns true if the user entered anything but 'n' or 'N'.
-    def make_blast_database?(file, type)
+    def make_blast_database?(action, file, type)
       puts
       puts
-      puts "FASTA file to format/reformat: #{file}"
+      puts "FASTA file to #{action}: #{file}"
       puts "FASTA type: #{type}"
       print 'Proceed? [y/n] (Default: y): '
       response = STDIN.gets.to_s.strip
@@ -159,20 +198,47 @@ module SequenceServer
       from_user.empty? && default || from_user
     end
 
+    # Check if a '.taxid_map.txt' file exists. If not, try getting it
+    # using blastdbcmd.
+    def taxid_map(db)
+      taxid_map = db.sub(/#{File.extname(db)}$/, '.taxid_map.txt')
+      if !File.exist?(taxid_map) || File.zero?(taxid_map)
+        extract_taxid_map(db, taxid_map)
+      end
+      "-taxid_map #{taxid_map}" if !File.zero?(taxid_map)
+    end
+
     # Get taxid from the user. Returns user input or 0.
     #
     # Using 0 as taxid is equivalent to not setting taxid for the database
     # that will be created.
-    def fetch_tax_id
-      default = 0
+    def taxid
       print 'Enter taxid (optional): '
       user_response = STDIN.gets.strip
-      user_response.empty? && default || Integer(user_response)
-    rescue
+      "-taxid #{user_response ? Integer(user_response) : 0}"
+    rescue ArgumentError # presumably from call to Interger()
       puts 'taxid should be a number'
       retry
     end
 
+    def _make_blast_database(file, type, title, taxonomy)
+      extract_fasta(file) unless File.exist?(file)
+      cmd = "makeblastdb -parse_seqids -hash_index -in #{file} " \
+            "-dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
+            " #{taxonomy}"
+      out, err = sys(cmd, path: config[:bin])
+      puts out.strip
+      puts err.strip
+    rescue CommandFailed => e
+      puts <<~MSG
+        Could not create BLAST database for: #{file}
+        Tried: #{cmd}
+        stdout: #{e.stdout}
+        stderr: #{e.stderr}
+      MSG
+      exit!
+    end
+
     # Extract FASTA file from BLAST database.
     #
     # Invoked while reformatting a BLAST database if the corresponding
@@ -192,10 +258,24 @@ module SequenceServer
       exit!
     end
 
+    def extract_taxid_map(db, taxmap_file)
+      cmd = "blastdbcmd -entry all -db #{db} -outfmt '%i %T'"
+      sys(cmd, stdout: taxmap_file, path: config[:bin])
+    rescue CommandFailed => e
+      # silence error
+    end
+
     # Returns true if the database name appears to be a multi-part database
     # name.
+    #
+    # e.g.
+    # /home/ben/pd.ben/sequenceserver/db/nr.00 => yes
+    # /home/ben/pd.ben/sequenceserver/db/nr => no
+    # /home/ben/pd.ben/sequenceserver/db/img3.5.finished.faa.01 => yes
+    # /home/ben/pd.ben/sequenceserver/db/nr00 => no
+    # /mnt/blast-db/refseq_genomic.100 => yes
     def multipart_database_name?(db_name)
-      Database.multipart_database_name? db_name
+      !(db_name.match(%r{.+/\S+\.\d{2,3}$}).nil?)
     end
 
     # Returns true if first character of the file is '>'.

data/lib/sequenceserver/routes.rb

@@ -91,7 +91,7 @@ module SequenceServer
     post '/' do
       if params[:input_sequence]
         @input_sequence = params[:input_sequence]
-        erb :layout
+        erb :search, layout: true
       else
         job = Job.create(params)
         redirect to("/#{job.id}")

data/lib/sequenceserver/version.rb

@@ -1,4 +1,4 @@
 # Define version number.
 module SequenceServer
-  VERSION = '2.0.0.rc7'.freeze
+  VERSION = '2.0.0.rc8'.freeze
 end

data/public/css/sequenceserver.css

@@ -330,7 +330,13 @@ td.nowrap-ellipsis {
 .notifications {
   position: relative;
   height: 32px;
+  /**
+   * Notifications should appear over the bottom border of the textarea, high
+   * enough that they do not interfere with the database listing below and
+   * narrow enough not to cover the textarea resize handle on the right. */
   top: -16px;
+  margin-right: 35px;
+  margin-left: 35px;
 }
 .notification {
   height: 32px !important;
@@ -403,7 +409,7 @@ input[type=checkbox] {
 }
 
 .sidebar li a {
-  padding: 3px 10px;
+  padding: 3px 2px;
 }
 
 .sidebar.affix {
@@ -552,4 +558,4 @@ input[type=checkbox] {
 /* CSS class to facilitate the hit header wrapping */
 .hit-header {
   display: table-cell;
-}
+}