sequenceserver 2.0.0.rc3 → 2.0.0.rc8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 44c3b5780875a23cdbaffe95e97b05da87c350c6e0cd54005b586127366a71d5
-  data.tar.gz: dd5d384d11283d7acf7bd8fb74c1b627249cab8a3feedda899e240e719c1b448
+  metadata.gz: 449efaa2619fd2c83a4116480c78ae6b892804f2f9bc23c3d074b8b0493985c5
+  data.tar.gz: 393f9da8bd3df53bff3ed3a137a5d9484ad5ec9cfa1de3c7b15c3f1a6833f092
 SHA512:
-  metadata.gz: 7f2a8521c17cd5f3132a0af043240c95a2f579d53229232401c8ac1791ee043d5e96790ac97fe1552cea8ea0b4ff0d139f1b4aaa11938838c16d3a2c6ce3fdf7
-  data.tar.gz: 8ea40d2d2b618d28a35f9555d4641ec363b72f8d3847ea1d7baa2c6134cd3e06914f5222919d6057b8fc10eb48275c2b18d932c7301591a94ea94f450e250e6a
+  metadata.gz: de2fdcef61a8fcdbe8907935a1f0b545219698fe9d37fcad6cd40aaa50e779010b7e91a9158eb76590baed78ce3743cdc173b664ff5d0969dfaca42eb6bd621c
+  data.tar.gz: a65c243d19435021b00a360ddfa8f617fa58e5944dd55117a8865f11dec4554a86e577cdfee3013a82638dc45de697a33156da1ba0a2fedd8e2936a7d760d5d9

data/bin/sequenceserver

@@ -40,7 +40,7 @@ def ask_to_join
       "-d ifq -d emailAddress=#{response} -d submit=Submit > /dev/null 2> /dev/null"
       system post_email_cmd
     end
-    system "touch #{asked_to_join}"
+    system "mkdir -p #{SequenceServer::DOTDIR} && touch #{asked_to_join}"
   end
 end
 
@@ -68,10 +68,6 @@ begin
         # of threads to use in config file.
         $ sequenceserver -s -n 16
 
-        # See if you have FASTA files in database dir that haven't
-        # been converted into BLAST database.
-        $ sequenceserver -u
-
         # Search for FASTA files in database dir that haven't been
         # converted into BLAST database yet, and convert them.
         $ sequenceserver -m
@@ -135,9 +131,6 @@ begin
     on 'l', 'list_databases',
        'List BLAST databases'
 
-    on 'u', 'list-unformatted-fastas',
-       'List unformatted FASTA files'
-
     on 'i', 'interactive',
        'Run SequenceServer in interactive mode'
 
@@ -285,9 +278,7 @@ begin
         fetch_option(:database_dir).value = response
         redo
       rescue SequenceServer::NO_BLAST_DATABASE_FOUND => e
-        unless list_databases? || list_unformatted_fastas? ||
-               make_blast_databases?
-
+        unless list_databases? || make_blast_databases?
           # Print error raised.
           puts
           puts e
@@ -305,18 +296,46 @@ begin
           unless response =~ /^[n]$/i
             puts
             puts 'Searching ...'
-            if SequenceServer::Database.unformatted_fastas.empty?
-              puts "Couldn't find any FASTA files."
-              exit!
-            else
-              formatted = SequenceServer::Database.make_blast_databases
+            if SequenceServer.makeblastdb.any_unformatted?
+              formatted = SequenceServer.makeblastdb.format
               exit! if formatted.empty? && !set?
               redo unless set?
+            else
+              puts "Couldn't find any FASTA files."
+              exit!
             end
           else
             exit! unless set?
           end
         end
+      rescue SequenceServer::INCOMPATIBLE_BLAST_DATABASES => e
+        unless list_databases? || make_blast_databases?
+          # Print error raised.
+          puts
+          puts e
+
+          # Offer user to format the FASTA files.
+          database_dir = SequenceServer.config[:database_dir]
+          puts
+          puts <<~MSG
+            Incompatible databases can cause BLAST searches and other features of
+            SequenceServer to fail unexpectedly.
+            You can view incompatible databases and choose to reformat them below.
+            Alternatively, please remove them from databases directory.
+
+            View incompatible databases? [y/n] (Default: y).
+          MSG
+          puts
+          print '>> '
+          response = STDIN.gets.to_s.strip
+          unless response =~ /^[n]$/i
+            reformatted = SequenceServer.makeblastdb.reformat
+            exit! if reformatted.empty? && !set?
+            redo unless set?
+          else
+            exit! unless set?
+          end
+        end
       rescue SequenceServer::ENOENT,
         SequenceServer::CONFIG_FILE_ERROR,
         SequenceServer::BLAST_DATABASE_ERROR,
@@ -361,22 +380,13 @@ begin
         exit
       end
 
-      if list_unformatted_fastas? || make_blast_databases?
-        unformatted_fastas = SequenceServer::Database.unformatted_fastas
-        if unformatted_fastas.empty?
+      if make_blast_databases?
+        if SequenceServer.makeblastdb.scan
+          SequenceServer.makeblastdb.run
+        else
           puts "All FASTA files in #{SequenceServer.config[:database_dir]} " \
                'are formatted.'
-          exit
         end
-      end
-
-      if list_unformatted_fastas?
-        puts unformatted_fastas
-        exit
-      end
-
-      if make_blast_databases?
-        SequenceServer::Database.make_blast_databases
         exit
       end
 

data/lib/sequenceserver.rb

@@ -4,8 +4,12 @@ require 'resolv'
 
 # Top level module / namespace.
 module SequenceServer
-  # Use a fixed minimum version of BLAST+
+  # The default version of BLAST that will be downloaded and configured for use.
   BLAST_VERSION = '2.10.0+'.freeze
+  # The minimum version of BLAST that SequenceServer is happy to run with. This
+  # is for compatiblity with older database formats. Users will download BLAST
+  # themselves.
+  MIN_BLAST_VERSION = '2.9.0+'.freeze
 
   # Default location of configuration file.
   DEFAULT_CONFIG_FILE = '~/.sequenceserver.conf'.freeze
@@ -20,6 +24,7 @@ module SequenceServer
   require 'sequenceserver/config'
   require 'sequenceserver/server'
   require 'sequenceserver/routes'
+  require 'sequenceserver/makeblastdb'
   require 'sequenceserver/job_remover'
   require 'sequenceserver/exceptions'
   require 'sequenceserver/sys'
@@ -53,6 +58,11 @@ module SequenceServer
                   end
     end
 
+    # MAKEBLASTDB service object.
+    def makeblastdb
+      @makeblastdb ||= MAKEBLASTDB.new(config[:database_dir])
+    end
+
     # SequenceServer initialisation routine.
     def init(config = {})
       # Use default config file if caller didn't specify one.
@@ -106,7 +116,6 @@ module SequenceServer
 
     # Run SequenceServer using WEBrick.
     def run
-      check_host
       Server.run(self)
     rescue Errno::EADDRINUSE
       puts "** Could not bind to port #{config[:port]}."
@@ -126,10 +135,17 @@ module SequenceServer
     def on_start
       puts '** SequenceServer is ready.'
       puts "   Go to #{server_url} in your browser and start BLASTing!"
-      puts '   To share your setup, please try one of the following: '
-      puts "     -  http://#{ip_address}:#{config[:port]}"
-      puts "     -  http://#{hostname}:#{config[:port]}" if hostname
-      puts '   Press CTRL+C to quit.'
+      if ip_address
+        puts '   To share your setup, try one of the following addresses. These'
+        puts '   may only work within your home, office, or university network.'
+        puts "     -  http://#{ip_address}:#{config[:port]}"
+        puts "     -  http://#{hostname}:#{config[:port]}" if hostname
+        puts '   To share your setup with anyone in the world, ask your IT team'
+        puts '   for a public IP address.'
+        puts '   To disable sharing, set :host: key in config file to 127.0.0.1'
+        puts '   and restart server.'
+      end
+      puts '   To terminate server, press CTRL+C'
       open_in_browser(server_url)
     end
 
@@ -186,7 +202,12 @@ module SequenceServer
       end
 
       logger.debug("Will look for BLAST+ databases in: #{config[:database_dir]}")
-      Database.scan_databases_dir
+
+      makeblastdb.scan
+      fail NO_BLAST_DATABASE_FOUND, config[:database_dir] if !makeblastdb.any_formatted?
+      fail INCOMPATIBLE_BLAST_DATABASES, config[:database_dir] if makeblastdb.any_incompatible?
+
+      Database.collection = makeblastdb.formatted_fastas
       Database.each do |database|
         logger.debug("Found #{database.type} database '#{database.title}'" \
                      " at '#{database.name}'")
@@ -204,16 +225,6 @@ module SequenceServer
       raise NUM_THREADS_INCORRECT
     end
 
-    # Check and warn user if host is 0.0.0.0 (default).
-    def check_host
-      # rubocop:disable Style/GuardClause
-      if config[:host] == '0.0.0.0'
-        logger.warn 'Will listen on all interfaces (0.0.0.0).' \
-                    ' Consider using 127.0.0.1 (--host option).'
-      end
-      # rubocop:enable Style/GuardClause
-    end
-
     def load_extension
       return unless config[:require]
 
@@ -234,7 +245,7 @@ module SequenceServer
       end
       version = out.split[1]
       fail BLAST_NOT_INSTALLED_OR_NOT_EXECUTABLE if version.empty?
-      fail BLAST_NOT_COMPATIBLE, version unless version == BLAST_VERSION
+      fail BLAST_NOT_COMPATIBLE, version unless is_compatible(version, MIN_BLAST_VERSION)
     end
 
     def server_url
@@ -243,13 +254,14 @@ module SequenceServer
       "http://#{host}:#{config[:port]}"
     end
 
-    # Returns a local ip adress
+    # Returns a local ip adress.
     def ip_address
-      Socket.ip_address_list.find { |ai| ai.ipv4? && !ai.ipv4_loopback? }.ip_address
+      addrinfo = Socket.ip_address_list.find { |ai| ai.ipv4? && !ai.ipv4_loopback? }
+      addrinfo.ip_address if addrinfo
     end
 
-    # Returns machine's hostname based on the local ip;
-    # If hostname cannot be determined then print nothing
+    # Returns machine's hostname based on the local ip. If hostname cannot be
+    # determined returns nil.
     def hostname
       Resolv.getname(ip_address) rescue nil
     end
@@ -285,5 +297,19 @@ module SequenceServer
     def command?(command)
       system("which #{command} > /dev/null 2>&1")
     end
+
+    # Returns true if the given version is higher than the minimum expected
+    # version string.
+    def is_compatible(given, expected)
+      # The speceship operator (<=>) below returns -1, 0, 1 depending on
+      # on whether the left operand is lower, same, or higher than the
+      # right operand. We want the left operand to be the same or higher.
+      (parse_version(given) <=> parse_version(expected)) >= 0
+    end
+
+    # Turn version string into an arrary of its component numbers.
+    def parse_version(version_string)
+      version_string.split('.').map(&:to_i)
+    end
   end
 end

data/lib/sequenceserver/blast/job.rb

@@ -65,6 +65,16 @@ module SequenceServer
           error = IO.foreach(stderr).grep(ERROR_LINE).join
           error = File.read(stderr) if error.empty?
           fail InputError, error
+        when 2
+          fail InputError, <<~MSG
+            BLAST signalled a problem with the databases that you searched.
+
+            Most likely one or more of your databases were created using an
+            older version of BLAST. Please consider recreating the databases
+            using BLAST #{BLAST_VERSION}.
+
+            As a temporary solution, you can try searching one database at a time.
+          MSG
         when 4
           # Out of memory. User can retry with a shorter search, so raising
           # InputError here instead of SystemError.
@@ -79,7 +89,7 @@ module SequenceServer
           # the job. This is a SystemError.
           fail SystemError, 'Ran out of disk space.'
         else
-          # I am not sure what the exit codes 2 & 3 means and we should note
+          # I am not sure what the exit codes 3 means and we should not
           # encounter exit code 5. The only other error that I know can happen
           # but is not yet handled is when BLAST+ binaries break such as after
           # macOS updates. So raise SystemError, include the exit status in the

data/lib/sequenceserver/database.rb

@@ -1,4 +1,3 @@
-require 'find'
 require 'open3'
 require 'digest/md5'
 require 'forwardable'
@@ -81,12 +80,15 @@ module SequenceServer
         @collection ||= {}
       end
 
-      private :collection
-
-      def <<(database)
-        collection[database.id] = database
+      def collection=(databases_attrs)
+        databases_attrs.each do |db_attrs|
+          db = Database.new(*db_attrs)
+          collection[db.id] = db
+        end
       end
 
+      private :collection
+
       def [](ids)
         ids = Array ids
         collection.values_at(*ids)
@@ -183,169 +185,6 @@ module SequenceServer
       def clear
         collection.clear
       end
-
-      # Recurisvely scan `database_dir` for blast databases.
-      #
-      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
-      def blastdbcmd
-        cmd = "blastdbcmd -recursive -list #{config[:database_dir]}" \
-              ' -list_outfmt "%f	%t	%p	%n	%l	%d"'
-        out, err = sys(cmd, path: config[:bin])
-        errpat = /BLAST Database error/
-        fail BLAST_DATABASE_ERROR.new(cmd, err) if err.match(errpat)
-        return out
-      rescue CommandFailed => e
-        fail BLAST_DATABASE_ERROR.new(cmd, e.stderr)
-      end
-
-      def scan_databases_dir
-        out = blastdbcmd
-        fail NO_BLAST_DATABASE_FOUND, config[:database_dir] if out.empty?
-        out.each_line do |line|
-          name = line.split('	')[0]
-          next if multipart_database_name?(name)
-          self << Database.new(*line.split('	'))
-        end
-      end
-      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
-
-      # Recursively scan `database_dir` for un-formatted FASTA and format them
-      # for use with BLAST+.
-      def make_blast_databases
-        unformatted_fastas.select do |file, sequence_type|
-          make_blast_database(file, sequence_type)
-        end
-      end
-
-      # Returns an Array of FASTA files that may require formatting, and the
-      # type of sequence contained in each FASTA.
-      #
-      #   > unformatted_fastas
-      #   => [['/foo/bar.fasta', :nulceotide], ...]
-      def unformatted_fastas
-        list = []
-        database_dir = config[:database_dir]
-        Find.find database_dir do |file|
-          next if File.directory? file
-          next if Database.include? file
-          next unless probably_fasta? file
-          sequence_type = guess_sequence_type_in_fasta file
-          if %i[protein nucleotide].include?(sequence_type)
-            list << [file, sequence_type]
-          end
-        end
-        list
-      end
-
-      # Create BLAST database, given FASTA file and sequence type in FASTA file.
-      def make_blast_database(file, type)
-        return unless make_blast_database? file, type
-        title = get_database_title(file)
-        taxid = fetch_tax_id
-        _make_blast_database(file, type, title, taxid)
-      end
-
-      def _make_blast_database(file, type, title, taxid, quiet = false)
-        cmd = 'makeblastdb -parse_seqids -hash_index ' \
-              "-in #{file} -dbtype #{type.to_s.slice(0, 4)} -title '#{title}'" \
-              " -taxid #{taxid}"
-        out, err = sys(cmd, path: config[:bin])
-        puts out, err unless quiet
-      end
-
-      # Show file path and guessed sequence type to the user and obtain a y/n
-      # response.
-      #
-      # Returns true if the user entered anything but 'n' or 'N'.
-      def make_blast_database?(file, type)
-        puts
-        puts
-        puts "FASTA file: #{file}"
-        puts "FASTA type: #{type}"
-        print 'Proceed? [y/n] (Default: y): '
-        response = STDIN.gets.to_s.strip
-        !response.match(/n/i)
-      end
-
-      # Generate a title for the given database and show it to the user for
-      # confirmation.
-      #
-      # Returns user input if any. Auto-generated title otherwise.
-      def get_database_title(path)
-        default = make_db_title(File.basename(path))
-        print "Enter a database title or will use '#{default}': "
-        from_user = STDIN.gets.to_s.strip
-        from_user.empty? && default || from_user
-      end
-
-      # Get taxid from the user. Returns user input or 0.
-      #
-      # Using 0 as taxid is equivalent to not setting taxid for the database
-      # that will be created.
-      def fetch_tax_id
-        default = 0
-        print 'Enter taxid (optional): '
-        user_response = STDIN.gets.strip
-        user_response.empty? && default || Integer(user_response)
-      rescue
-        puts 'taxid should be a number'
-        retry
-      end
-
-      # Returns true if the database name appears to be a multi-part database
-      # name.
-      #
-      # e.g.
-      # /home/ben/pd.ben/sequenceserver/db/nr.00 => yes
-      # /home/ben/pd.ben/sequenceserver/db/nr => no
-      # /home/ben/pd.ben/sequenceserver/db/img3.5.finished.faa.01 => yes
-      # /home/ben/pd.ben/sequenceserver/db/nr00 => no
-      # /mnt/blast-db/refseq_genomic.100 => yes
-      def multipart_database_name?(db_name)
-        !(db_name.match(%r{.+/\S+\.\d{2,3}$}).nil?)
-      end
-
-      # Returns true if first character of the file is '>'.
-      def probably_fasta?(file)
-        File.read(file, 1) == '>'
-      end
-
-      # Suggests improved titles when generating database names from files
-      # for improved apperance and readability in web interface.
-      # For example:
-      # Cobs1.4.proteins.fasta -> Cobs 1.4 proteins
-      # S_invicta.xx.2.5.small.nucl.fa -> S invicta xx 2.5 small nucl
-      def make_db_title(db_name)
-        db_name.tr!('"', "'")
-        # removes .fasta like extension names
-        db_name.gsub!(File.extname(db_name), '')
-        # replaces _ with ' ',
-        db_name.gsub!(/(_)/, ' ')
-        # replaces '.' with ' ' when no numbers are on either side,
-        db_name.gsub!(/(\D)\.(?=\D)/, '\1 ')
-        # preserves version numbers
-        db_name.gsub!(/\W*(\d+([.-]\d+)+)\W*/, ' \1 ')
-        db_name
-      end
-
-      # Guess whether FASTA file contains protein or nucleotide sequences by
-      # sampling a few few characters of the file.
-      def guess_sequence_type_in_fasta(file)
-        sequences = sample_sequences(file)
-        sequence_types = sequences.map { |seq| Sequence.guess_type(seq) }
-        sequence_types = sequence_types.uniq.compact
-        (sequence_types.length == 1) && sequence_types.first
-      end
-
-      # Read first 1,048,576 characters of the file, split the read text on
-      # fasta def line pattern and return.
-      #
-      # If the given file is FASTA, returns Array of as many different
-      # sequences in the portion of the file read. Returns the portion
-      # of the file read wrapped in an Array otherwise.
-      def sample_sequences(file)
-        File.read(file, 1_048_576).split(/^>.+$/).delete_if(&:empty?)
-      end
     end
   end
 end