sequenceserver 2.0.0.beta4 → 2.0.0.rc5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a8617ffb07710b984f84dfc0dfb7d43c66cac4c3
-  data.tar.gz: 14ddad09e8dcb15793d8e06227138ee4c862b280
+SHA256:
+  metadata.gz: b04c2f43509bf7d4a15d555e3bd9cea86d3c7770c7a3e35016f963f877055ecf
+  data.tar.gz: 25678397eacb2c3f445902624baff80a90d2a47ca98d2b703e13de3eca84cf1e
 SHA512:
-  metadata.gz: c130d6e3828eae5dae108786e08433d5641112305962884dc2dada918a6a92c17632f7c46400c58ea0adabf106cb02e348bf6adc35dc8cbf296300081221fc99
-  data.tar.gz: 17a41dcb923c2e64337b960de1e3717b458077a9658a9e7785ab5815c43860aff09033aacd7812146e0f1fe368878f649794b1aa5bd8bbaf13c8a4a34fe7b0ac
+  metadata.gz: a8e6a4c1b4917d9bd35d07ef8e580e1b75105bfd1709fb892d9ed523da91f8c3a08d50c1c9da304530cd0bb5d77da00a4944c65cd5ce90fc979c9d6099149058
+  data.tar.gz: ef493ec4cf1aef63b2f535251143b34154d0d332a47a1d15668c7a74b139d5e6c2a2f9c33b5c7072ae9c2fbbcf33048fad4e828273c85a199a6ab79874d6e766

data/.dockerignore

@@ -0,0 +1 @@
+_site

data/.travis.yml

@@ -24,14 +24,17 @@ before_install:
   - wget -c "https://github.com/mozilla/geckodriver/releases/download/v0.24.0/geckodriver-v0.24.0-linux64.tar.gz" && tar xvf geckodriver-*.tar.gz -C bin
   # Download codeclimate test reporter to bin and make it executable
   - wget -c "https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64" -O bin/cc-test-reporter && chmod +x bin/cc-test-reporter
-  # Download and extract NCBI BLAST+ 2.9.0
-  - wget -c "ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.9.0/ncbi-blast-2.9.0+-x64-linux.tar.gz" && tar xvf ncbi-blast-*.tar.gz
+  # Download and extract NCBI BLAST+ 2.10.0
+  - wget -c "ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.10.0/ncbi-blast-2.10.0+-x64-linux.tar.gz" && tar xvf ncbi-blast-*.tar.gz
   # Conclude installation by adding opt/bin/ containing codeclimate test reporter
-  # and geckodriver, and opt/ncbi-blast-2.9.0+/bin containing BLAST+ binaries to
+  # and geckodriver, and opt/ncbi-blast-2.10.0+/bin containing BLAST+ binaries to
   # PATH.
-  - export PATH="$PWD/bin:$PWD/ncbi-blast-2.9.0+/bin:$PATH"
+  - export PATH="$PWD/bin:$PWD/ncbi-blast-2.10.0+/bin:$PATH"
   # Reset working directory or subsequent steps may fail.
   - cd ..
+  # Update bundler. Travis was using bundler 1.16.2 at the time of this writing
+  # and the builds were failing because Gemfile.lock requires bundler > 2.0
+  - gem install bundler -v 2.1.4
 
 env:
   global:

data/AppImage/sequenceserver.sh

@@ -1,3 +1,8 @@
+# cd back to the directory where the AppImage was launched from. AppRun changes
+# the working directory so as to work with binaries that have /usr hard-coded;
+# we don't.
+cd $OWD
+
 # Set GEM_PATH to where we installed gems in the AppImage.
 export GEM_PATH=$APPDIR/var/lib/gems/2.3.0
 

data/Dockerfile

@@ -1,23 +1,25 @@
-FROM debian:stretch-backports
+FROM debian:buster-slim
 
 LABEL Description="Intuitive local web frontend for the BLAST bioinformatics tool"
 LABEL MailingList="https://groups.google.com/forum/#!forum/sequenceserver"
 LABEL Website="http://www.sequenceserver.com"
-LABEL Version="1.1.0 beta"
 
-RUN apt-get update  && apt-get install -y --no-install-recommends \
-        build-essential \
-        ruby ruby-dev \
-        curl wget \
-        gnupg \
-        git \
-        zlib1g-dev
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        ruby ruby-dev build-essential curl gnupg git wget \
+        zlib1g-dev && rm -rf /var/lib/apt/lists/*
 
 VOLUME ["/db"]
 EXPOSE 4567
 
 COPY . /sequenceserver
 WORKDIR /sequenceserver
-RUN gem install bundler && bundle install --without=development
-RUN yes '' | bundle exec bin/sequenceserver -s
-ENTRYPOINT ["bundle", "exec", "bin/sequenceserver", "-d", "/db"]
+# Install bundler, then use bundler to install SequenceServer's dependencies,
+# and then use SequenceServer to install BLAST. In the last step, -s is used
+# so that SequenceServer will exit after writing configuration file instead
+# of starting up, while -d is used to suppress questions about database dir.
+RUN gem install bundler && \
+        bundle install --without=development && \
+        yes '' | bundle exec bin/sequenceserver -s -d spec/database/sample
+RUN touch ~/.sequenceserver/asked_to_join
+
+CMD ["bundle", "exec", "bin/sequenceserver", "-d", "/db"]

data/bin/sequenceserver

@@ -20,6 +20,30 @@ def download_from_url(url)
   system(cmd)
 end
 
+def ask_to_join
+  asked_to_join = File.join(SequenceServer::DOTDIR, 'asked_to_join')
+  unless File.exist?(asked_to_join)
+    puts
+    puts <<~MSG
+    Do you want to be notified of SequenceServer releases and any
+    other important announcements (3-12 messages a year)? If yes,
+    please provide your email address below or press enter to
+    continue (you won't be prompted again).
+    MSG
+    print '>> '
+    response = STDIN.gets.to_s.strip
+    puts
+
+    if response =~ /@/
+      post_email_cmd = "curl -m 5 https://docs.google.com/forms/u/0/d/e/" \
+      "1FAIpQLSe7sOCiKzYbI7LwErid-g3wfIU5Zpi6VTm0ILJyR036RqC8zg/formResponse " \
+      "-d ifq -d emailAddress=#{response} -d submit=Submit > /dev/null 2> /dev/null"
+      system post_email_cmd
+    end
+    system "touch #{asked_to_join}"
+  end
+end
+
 begin
   Slop.parse!(strict: true, help: true) do
     banner <<~BANNER
@@ -44,10 +68,6 @@ begin
         # of threads to use in config file.
         $ sequenceserver -s -n 16
 
-        # See if you have FASTA files in database dir that haven't
-        # been converted into BLAST database.
-        $ sequenceserver -u
-
         # Search for FASTA files in database dir that haven't been
         # converted into BLAST database yet, and convert them.
         $ sequenceserver -m
@@ -111,9 +131,6 @@ begin
     on 'l', 'list_databases',
        'List BLAST databases'
 
-    on 'u', 'list-unformatted-fastas',
-       'List unformatted FASTA files'
-
     on 'i', 'interactive',
        'Run SequenceServer in interactive mode'
 
@@ -261,8 +278,7 @@ begin
         fetch_option(:database_dir).value = response
         redo
       rescue SequenceServer::NO_BLAST_DATABASE_FOUND => e
-        unless list_databases? || list_unformatted_fastas? ||
-               make_blast_databases?
+        unless list_databases? || make_blast_databases?
 
           # Print error raised.
           puts
@@ -281,13 +297,13 @@ begin
           unless response =~ /^[n]$/i
             puts
             puts 'Searching ...'
-            if SequenceServer::Database.unformatted_fastas.empty?
-              puts "Couldn't find any FASTA files."
-              exit!
-            else
-              formatted = SequenceServer::Database.make_blast_databases
+            if SequenceServer.makeblastdb.scan
+              formatted = SequenceServer.makeblastdb.run
               exit! if formatted.empty? && !set?
               redo unless set?
+            else
+              puts "Couldn't find any FASTA files."
+              exit!
             end
           else
             exit! unless set?
@@ -337,22 +353,13 @@ begin
         exit
       end
 
-      if list_unformatted_fastas? || make_blast_databases?
-        unformatted_fastas = SequenceServer::Database.unformatted_fastas
-        if unformatted_fastas.empty?
+      if make_blast_databases?
+        if SequenceServer.makeblastdb.scan
+          SequenceServer.makeblastdb.run
+        else
           puts "All FASTA files in #{SequenceServer.config[:database_dir]} " \
                'are formatted.'
-          exit
         end
-      end
-
-      if list_unformatted_fastas?
-        puts unformatted_fastas
-        exit
-      end
-
-      if make_blast_databases?
-        SequenceServer::Database.make_blast_databases
         exit
       end
 
@@ -379,6 +386,8 @@ begin
 
       SequenceServer.config.write_config_file if fetch_option(:set).value
 
+      ask_to_join
+
       SequenceServer.run
     end
   end
@@ -386,4 +395,4 @@ rescue Slop::Error => e
   puts e
   puts "Run '#{$PROGRAM_NAME} -h' for help with command line options."
   exit
-end
+end

data/lib/sequenceserver.rb

@@ -4,8 +4,12 @@ require 'resolv'
 
 # Top level module / namespace.
 module SequenceServer
-  # Use a fixed minimum version of BLAST+
-  BLAST_VERSION = '2.9.0+'.freeze
+  # The default version of BLAST that will be downloaded and configured for use.
+  BLAST_VERSION = '2.10.0+'.freeze
+  # The minimum version of BLAST that SequenceServer is happy to run with. This
+  # is for compatiblity with older database formats. Users will download BLAST
+  # themselves.
+  MIN_BLAST_VERSION = '2.9.0+'.freeze
 
   # Default location of configuration file.
   DEFAULT_CONFIG_FILE = '~/.sequenceserver.conf'.freeze
@@ -20,6 +24,7 @@ module SequenceServer
   require 'sequenceserver/config'
   require 'sequenceserver/server'
   require 'sequenceserver/routes'
+  require 'sequenceserver/makeblastdb'
   require 'sequenceserver/job_remover'
   require 'sequenceserver/exceptions'
   require 'sequenceserver/sys'
@@ -53,6 +58,11 @@ module SequenceServer
                   end
     end
 
+    # MAKEBLASTDB service object.
+    def makeblastdb
+      @makeblastdb ||= MAKEBLASTDB.new(config[:database_dir])
+    end
+
     # SequenceServer initialisation routine.
     def init(config = {})
       # Use default config file if caller didn't specify one.
@@ -138,10 +148,14 @@ module SequenceServer
       puts
       puts '** Thank you for using SequenceServer :).'
       puts '   Please cite: '
-      puts '       Priyam A, Woodcroft BJ, Rai V, Munagala A, Moghul I, Ter F,'
-      puts '       Gibbins MA, Moon H, Leonard G, Rumpf W & Wurm Y. 2015.'
-      puts '       Sequenceserver: a modern graphical user interface for'
-      puts '       custom BLAST databases. biorxiv doi: 10.1101/033142.'
+      puts '       Priyam A, Woodcroft BJ, Rai V, Moghul I, Munagala A, Ter F,'
+      puts '       Chowdhary H, Pieniak I, Maynard LJ, Gibbins MA, Moon H,'
+      puts '       Davis-Richardson A, Uludag M, Watson-Haigh N, Challis R,'
+      puts '       Nakamura H, Favreau E, Gómez EA, Pluskal T, Leonard G,'
+      puts '       Rumpf W & Wurm Y.'
+      puts '       Sequenceserver: A modern graphical user interface for'
+      puts '       custom BLAST databases.'
+      puts '       Molecular Biology and Evolution (2019)'
     end
 
     # This method is invoked by the -i switch to start an IRB shell with
@@ -230,7 +244,7 @@ module SequenceServer
       end
       version = out.split[1]
       fail BLAST_NOT_INSTALLED_OR_NOT_EXECUTABLE if version.empty?
-      fail BLAST_NOT_COMPATIBLE, version unless version == BLAST_VERSION
+      fail BLAST_NOT_COMPATIBLE, version unless is_compatible(version, MIN_BLAST_VERSION)
     end
 
     def server_url
@@ -281,5 +295,19 @@ module SequenceServer
     def command?(command)
       system("which #{command} > /dev/null 2>&1")
     end
+
+    # Returns true if the given version is higher than the minimum expected
+    # version string.
+    def is_compatible(given, expected)
+      # The speceship operator (<=>) below returns -1, 0, 1 depending on
+      # on whether the left operand is lower, same, or higher than the
+      # right operand. We want the left operand to be the same or higher.
+      (parse_version(given) <=> parse_version(expected)) >= 0
+    end
+
+    # Turn version string into an arrary of its component numbers.
+    def parse_version(version_string)
+      version_string.split('.').map(&:to_i)
+    end
   end
 end

data/lib/sequenceserver/blast/job.rb

@@ -13,7 +13,6 @@ module SequenceServer
             # itself is self-contained. This will help with tests among
             # other things.
             FileUtils.cp(params[:xml], dir)
-            @advanced_params = {}
             @databases = []
             done!
           end
@@ -23,17 +22,19 @@ module SequenceServer
             @method    = params[:method]
             @qfile     = store('query.fa', params[:sequence])
             @databases = Database[params[:databases]]
-            @options   = params[:advanced].to_s.strip + defaults
-            @advanced_params = parse_advanced params[:advanced]
+            @advanced  = params[:advanced].to_s.strip
+            @options   = @advanced + defaults
           end
         end
       end
 
-      attr_reader :advanced_params
-
       # :nodoc:
       # Attributes used by us - should be considered private.
-      attr_reader :method, :qfile, :databases, :options
+      attr_reader :method, :qfile, :databases, :advanced, :options
+
+      # :nodoc:
+      # Deprecated; see Report#extract_params
+      attr_reader :advanced_params
 
       # :nodoc:
       # Returns path to the imported xml file if the job was created using the
@@ -64,6 +65,16 @@ module SequenceServer
           error = IO.foreach(stderr).grep(ERROR_LINE).join
           error = File.read(stderr) if error.empty?
           fail InputError, error
+        when 2
+          fail InputError, <<~MSG
+            BLAST signalled a problem with the databases that you searched.
+
+            Most likely one or more of your databases were created using an
+            older version of BLAST. Please consider recreating the databases
+            using BLAST #{BLAST_VERSION}.
+
+            As a temporary solution, you can try searching one database at a time.
+          MSG
         when 4
           # Out of memory. User can retry with a shorter search, so raising
           # InputError here instead of SystemError.
@@ -78,7 +89,7 @@ module SequenceServer
           # the job. This is a SystemError.
           fail SystemError, 'Ran out of disk space.'
         else
-          # I am not sure what the exit codes 2 & 3 means and we should note
+          # I am not sure what the exit codes 3 means and we should not
           # encounter exit code 5. The only other error that I know can happen
           # but is not yet handled is when BLAST+ binaries break such as after
           # macOS updates. So raise SystemError, include the exit status in the
@@ -94,24 +105,6 @@ module SequenceServer
 
       private
 
-      def parse_advanced(param_line)
-        param_list = (param_line || '').split(' ')
-        res = {}
-
-        param_list.each_with_index do |word, i|
-          nxt = param_list[i + 1]
-          if word.start_with? '-'
-            word.sub!('-', '')
-            unless nxt.nil? || nxt.start_with?('-')
-              res[word] = nxt
-            else
-              res[word] = 'True'
-            end
-          end
-        end
-        res
-      end
-
       def validate(params)
         validate_method params[:method]
         validate_sequences params[:sequence]

data/lib/sequenceserver/blast/report.rb

@@ -24,34 +24,15 @@ module SequenceServer
     class Report < Report
       def initialize(job)
         super do
-          @querydb = job.databases
           @queries = []
         end
       end
 
-      # Attributes parsed out from XML output.
-      attr_reader :program, :program_version, :params, :stats, :queries
-
-      # This is obtained from the job object.
-      attr_reader :querydb
+      # Attributes parsed out from BLAST output.
+      attr_reader :program, :program_version, :stats, :queries
 
-      # Returns database type (nucleotide or protein) used for running BLAST
-      # search. If we ran the BLAST search, this information is available
-      # from Job#databases. For imported XML, this is inferred from
-      # Report#program (i.e., the BLAST algorithm)
-      def dbtype
-        return @dbtype if @dbtype
-        @dbtype = if @querydb.empty?
-                    case program
-                    when /blastn|tblastn|tblastx/
-                      'nucleotide'
-                    when /blastp|blastx/
-                      'protein'
-                    end
-                  else
-                    @querydb.first.type
-                  end
-      end
+      # Attributes parsed from job metadata and BLAST output.
+      attr_reader :querydb, :dbtype, :params
 
       def to_json
         [:querydb, :program, :program_version, :params, :stats,
@@ -60,7 +41,8 @@ module SequenceServer
           h
         }.update(search_id: job.id,
                  submitted_at: job.submitted_at.utc,
-                 imported_xml: !!job.imported_xml_file).to_json
+                 imported_xml: !!job.imported_xml_file,
+                 seqserv_version: SequenceServer::VERSION).to_json
       end
 
       private
@@ -68,6 +50,8 @@ module SequenceServer
       # Generate report.
       def generate
         job.raise!
+        xml_ir = nil
+        tsv_ir = nil
         if job.imported_xml_file
           xml_ir = parse_xml File.read(job.imported_xml_file)
           tsv_ir = Hash.new do |h1,k1|
@@ -75,18 +59,15 @@ module SequenceServer
               h2[k2]=['','',[]]
             end
           end
-          extract_program_info xml_ir
-          extract_params xml_ir
-          extract_stats xml_ir
-          extract_queries xml_ir, tsv_ir
         else
           xml_ir = parse_xml File.read(Formatter.run(job, 'xml').file)
-          tsv_ir = parse_tsv File.read(Formatter.run(job, 'custom_tsv').file )
-          extract_program_info xml_ir
-          extract_params xml_ir
-          extract_stats xml_ir
-          extract_queries xml_ir, tsv_ir
+          tsv_ir = parse_tsv File.read(Formatter.run(job, 'custom_tsv').file)
         end
+        extract_program_info xml_ir
+        extract_db_info xml_ir
+        extract_params xml_ir
+        extract_stats xml_ir
+        extract_queries xml_ir, tsv_ir
       end
 
       # Make program name and program name + version available via `program`
@@ -96,6 +77,20 @@ module SequenceServer
         @program_version = ir[1]
       end
 
+      # Get database information (title and type) from job yaml or from XML.
+      # Sets `querydb` and `dbtype` attributes.
+      def extract_db_info(ir)
+        if job.databases.empty?
+          @querydb = ir[3].split.map do |path|
+            { title: File.basename(path) }
+          end
+          @dbtype = dbtype_from_program
+        else
+          @querydb = job.databases
+          @dbtype = @querydb.first.type
+        end
+      end
+
       # Make search params available via `params` attribute.
       #
       # Search params tweak the results. Like evalue cutoff or penalty to open
@@ -103,11 +98,20 @@ module SequenceServer
       # matrix, evalue, gapopen, gapextend, and filters are available from XML
       # output.
       def extract_params(ir)
+        # Parse/get params from the job first.
+        job_params = parse_advanced(job.advanced)
+        # Old jobs from beta releases may not have the advanced key but they
+        # will have the deprecated advanced_params key.
+        job_params.update(job.advanced_params) if job.advanced_params
+
+        # Parse params from BLAST XML.
         @params = Hash[
           *ir[7].first.map { |k, v| [k.gsub('Parameters_', ''), v] }.flatten
         ]
         @params['evalue'] = @params.delete('expect')
-        @params = job.advanced_params.merge(@params)
+
+        # Merge into job_params.
+        @params = job_params.merge(@params)
       end
 
       # Make search stats available via `stats` attribute.
@@ -232,6 +236,36 @@ module SequenceServer
         end
         ir
       end
+
+      # Parse BLAST CLI string from job.advanced.
+      def parse_advanced(param_line)
+        param_list = (param_line || '').split(' ')
+        res = {}
+
+        param_list.each_with_index do |word, i|
+          nxt = param_list[i + 1]
+          if word.start_with? '-'
+            word.sub!('-', '')
+            unless nxt.nil? || nxt.start_with?('-')
+              res[word] = nxt
+            else
+              res[word] = 'True'
+            end
+          end
+        end
+        res
+      end
+
+      # Returns database type (nucleotide or protein) inferred from
+      # Report#program (i.e., the BLAST algorithm)
+      def dbtype_from_program
+        case program
+        when /blastn|tblastn|tblastx/
+          'nucleotide'
+        when /blastp|blastx/
+          'protein'
+        end
+      end
     end
   end
 end