sequenceserver 2.1.0 → 3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3cf703fae66ad96d285ed0556c387dabb0d69be120db5bdd3c3f9f8a03d3a814
-  data.tar.gz: 503bb0b69cbf773ab536d99f6782f4fff19f4872c49e6377d49a9f7820861c3b
+  metadata.gz: 68dcb3dda53edae5d095423c164568484e786b927fcf733a4722eafde7c3a155
+  data.tar.gz: afb985fe5e762b8a1ab9e1dc4e8dd0bc6158f601a3cc95c0029f03a8587c1fa2
 SHA512:
-  metadata.gz: 1896b2afddf6033c98747f7eba1dabbe15b270940bd19760ebef4ed75af267760d45d40215663173ccb2f964472b458179a459316b86ef72bc26aa75d62faad0
-  data.tar.gz: 4ac1d9a5a205d299af4b1467521a9bc539e3833d0a571cfa90b490b19df958fc0f7ee2e1341e3d8d5e19fccbe2ee6874884fc7fbcbfbdc6abbe1e60df3c9a76e
+  metadata.gz: fb122a17858e6b7c567a418bda05358ae1a9b34ef1dfdcc11bb5f83097b47e427d7344e61e9f9102488f59f6448592b7d66e1e67ee213498e750dfea5e2e7c8f
+  data.tar.gz: b7b7504faf439acda18aafa0866e74f3645e431b4252bdb12ffd4896d7cf6ee065756db9e1bba027184794f306ce3b732c4911dce56d8b4a96e734dcf86c336c

data/COPYRIGHT.txt

@@ -3,7 +3,7 @@ SequenceServer is copyright Anurag Priyam, Ben J Woodcroft and Yannick Wurm,
 version 3, the text of which can be found in LICENSE.txt.
 
 Components of SequenceServer, including Sinatra, Ox, Slop, html5shiv,
-Underscore, jQuery, jQuery UI, Bootstrap and Webshim, are licensed under
+Underscore, jQuery, jQuery UI and Bootstrap, are licensed under
 the MIT license. D3.js is licensed under BSD license. biojs-vis-sequence
 is licensed under Apache license. Thin and JSON are licensed under Ruby
 license. All unmodified files from these and other sources retain their

data/bin/sequenceserver

@@ -119,6 +119,11 @@ begin
        'Port to run SequenceServer on',
        argument: true
 
+    on 'o', 'optimistic=',
+       'Optimistic mode does not perform database compatibility checks, which can be faster. ' \
+       'Only use this if you format FASTAs with "sequenceserver -make-blast-databases"',
+       argument: true
+
     on 's', 'set',
        'Set configuration value in default or given config file'
 
@@ -296,7 +301,7 @@ begin
           unless response =~ /^[n]$/i
             puts
             puts 'Searching ...'
-            if SequenceServer.makeblastdb.any_unformatted?
+            if SequenceServer.makeblastdb.any_to_format?
               formatted = SequenceServer.makeblastdb.format
               exit! if formatted.empty? && !set?
               redo unless set?
@@ -353,7 +358,9 @@ begin
       end
 
       if make_blast_databases?
-        if SequenceServer.makeblastdb.scan
+        if SequenceServer.makeblastdb.no_fastas?
+          puts "Couldn't find any FASTA files in #{SequenceServer.config[:database_dir]}."
+        elsif SequenceServer.makeblastdb.any_to_format_or_reformat?
           puts
           puts <<~MSG
           SequenceServer has scanned your databases directory and will now offer
@@ -375,7 +382,7 @@ begin
           print '>> '
           response = STDIN.gets.to_s.strip
           SequenceServer.makeblastdb.run unless response =~ /^[n]$/i
-          else
+        else
           puts "All FASTA files in #{SequenceServer.config[:database_dir]} " \
                'are formatted.'
         end

data/lib/sequenceserver/blast/error.rb

@@ -0,0 +1,53 @@
+# http://www.ncbi.nlm.nih.gov/books/NBK1763/ (Appendices)
+
+module SequenceServer
+  module BLAST
+    class Error
+      attr_reader :exitstatus, :stdout, :stderr
+
+      def initialize(exitstatus:, stdout:, stderr:)
+        @exitstatus = exitstatus
+        @stdout = stdout
+        @stderr = stderr
+      end
+
+      def raise!
+        return true if exitstatus.zero? && !File.zero?(stdout)
+
+        case exitstatus
+        when 1..2
+          # 1: Error in query sequences or options.
+          # 2: Error in BLAST databases.
+          error = IO.foreach(stderr).grep(ERROR_LINE).join
+          error = File.read(stderr) if error.empty?
+          fail InputError, "(#{exitstatus}) #{error}"
+        when 4
+          # Out of memory. User can retry with a shorter search, so raising
+          # InputError here instead of SystemError.
+          fail InputError, <<~MSG
+            Ran out of memory. Please try a smaller query, fewer and smaller
+            databases, or limiting the output by using advanced options.
+          MSG
+        when 6
+          # Error creating output files. It can't be a permission issue as that
+          # would have been caught while creating job directory. But we can run
+          # out of storage after creating the job directory and while running
+          # the job. This is a SystemError.
+          fail SystemError, 'Ran out of disk space.'
+        else
+          # I am not sure what the exit codes 3 means and we should not
+          # encounter exit code 5. The only other error that I know can happen
+          # but is not yet handled is when BLAST+ binaries break such as after
+          # macOS updates. So raise SystemError, include the exit status in the
+          # message, and say that that the "most likely" reason is broken BLAST+
+          # binaries.
+
+          error = File.read(stderr)
+          error = 'Most likely there is a problem with the BLAST+ binaries.' if error.empty?
+
+          fail SystemError, "BLAST failed abruptly (exit status: #{exitstatus}). #{error}"
+        end
+      end
+    end
+  end
+end

data/lib/sequenceserver/blast/formatter.rb

@@ -25,23 +25,32 @@ module SequenceServer
 
       attr_reader :format, :mime, :specifiers
 
-      def file
-        @file ||= File.join(job.dir, filename)
+      def filepath
+        @filepath ||= File.join(job.dir, filename)
+      end
+
+      def size
+        File.size(filepath)
       end
 
       def filename
         @filename ||= "sequenceserver-#{type}_report.#{mime}"
       end
 
+      def read_file
+        File.read(filepath)
+      end
+
       private
 
       attr_reader :job, :type
 
       def run
-        return if File.exist?(file)
+        return if File.exist?(filepath)
+
         command = "blast_formatter -archive '#{job.stdout}'" \
           " -outfmt '#{format} #{specifiers}'"
-        sys(command, path: config[:bin], dir: DOTDIR, stdout: file)
+        sys(command, path: config[:bin], dir: DOTDIR, stdout: filepath)
       rescue CommandFailed => e
         # Mostly we will never get here: empty archive file,
         # file permissions, broken BLAST binaries, etc. will

data/lib/sequenceserver/blast/job.rb

@@ -1,5 +1,6 @@
 require 'sequenceserver/job'
 require 'sequenceserver/zip_file_generator'
+require 'sequenceserver/blast/error'
 
 module SequenceServer
   # BLAST module.
@@ -56,46 +57,8 @@ module SequenceServer
                      " -query '#{qfile}' #{options}"
       end
 
-      # Override Job#raise! to raise specific API errors based on exitstatus
-      # and using contents of stderr to provide context about the error.
-      #
       def raise!
-        # Return true exit status is 0 and stdout is not empty.
-        return true if exitstatus.zero? && !File.zero?(stdout)
-
-        # Handle error. See [1].
-        case exitstatus
-        when 1..2
-          # 1: Error in query sequences or options.
-          # 2: Error in BLAST databases.
-          error = IO.foreach(stderr).grep(ERROR_LINE).join
-          error = File.read(stderr) if error.empty?
-          fail InputError, "(#{exitstatus}) #{error}"
-        when 4
-          # Out of memory. User can retry with a shorter search, so raising
-          # InputError here instead of SystemError.
-          fail InputError, <<~MSG
-            Ran out of memory. Please try a smaller query, fewer and smaller
-            databases, or limiting the output by using advanced options.
-          MSG
-        when 6
-          # Error creating output files. It can't be a permission issue as that
-          # would have been caught while creating job directory. But we can run
-          # out of storage after creating the job directory and while running
-          # the job. This is a SystemError.
-          fail SystemError, 'Ran out of disk space.'
-        else
-          # I am not sure what the exit codes 3 means and we should not
-          # encounter exit code 5. The only other error that I know can happen
-          # but is not yet handled is when BLAST+ binaries break such as after
-          # macOS updates. So raise SystemError, include the exit status in the
-          # message, and say that that the "most likely" reason is broken BLAST+
-          # binaries.
-          fail SystemError, <<~MSG
-            BLAST failed abruptly (exit status: #{exitstatus}). Most likely there is a
-            problem with the BLAST+ binaries.
-          MSG
-        end
+        SequenceServer::BLAST::Error.new(exitstatus: exitstatus, stdout: stdout, stderr: stderr).raise!
       end
 
       # Use it with a block to get a self-cleaning temporary archive file
@@ -189,7 +152,3 @@ module SequenceServer
     end
   end
 end
-
-# References
-# ----------
-# [1]: http://www.ncbi.nlm.nih.gov/books/NBK1763/ (Appendices)

data/lib/sequenceserver/blast/report.rb

@@ -36,6 +36,8 @@ module SequenceServer
       attr_reader :querydb, :dbtype, :params
 
       def to_json(*_args)
+        generate
+
         %i[querydb program program_version params stats
            queries].inject({}) do |h, k|
           h[k] = send(k)
@@ -48,10 +50,18 @@ module SequenceServer
                    non_parse_seqids: !!job.databases&.any?(&:non_parse_seqids?)).to_json
       end
 
-      private
+      def xml_file_size
+        return File.size(job.imported_xml_file) if job.imported_xml_file
+
+        generate
+
+        xml_formatter.size
+      end
 
       # Generate report.
       def generate
+        return self if @_generated
+
         job.raise!
         xml_ir = nil
         tsv_ir = nil
@@ -63,14 +73,34 @@ module SequenceServer
             end
           end
         else
-          xml_ir = parse_xml File.read(Formatter.run(job, 'xml').file)
-          tsv_ir = parse_tsv File.read(Formatter.run(job, 'custom_tsv').file)
+          xml_ir = parse_xml(xml_formatter.read_file)
+          tsv_ir = parse_tsv(tsv_formatter.read_file)
         end
         extract_program_info xml_ir
         extract_db_info xml_ir
         extract_params xml_ir
         extract_stats xml_ir
         extract_queries xml_ir, tsv_ir
+
+        @_generated = true
+
+        self
+      end
+
+      def done?
+        return true if job.imported_xml_file
+
+        File.exist?(xml_formatter.filepath) && File.exist?(tsv_formatter.filepath)
+      end
+
+      private
+
+      def xml_formatter
+        @xml_formatter ||= Formatter.run(job, 'xml')
+      end
+
+      def tsv_formatter
+        @tsv_formatter ||= Formatter.run(job, 'custom_tsv')
       end
 
       # Make program name and program name + version available via `program`

data/lib/sequenceserver/config.rb

@@ -131,7 +131,10 @@ module SequenceServer
         num_jobs: 1,
         job_lifetime: 43_200,
         # Set cloud_share_url to 'disabled' to disable the cloud sharing feature
-        cloud_share_url: 'https://share.sequenceserver.com/api/v1/shared-job'
+        cloud_share_url: 'https://share.sequenceserver.com/api/v1/shared-job',
+        # Warn in the UI before rendering results larger than this value
+        large_result_warning_threshold: 250 * 1024 * 1024,
+        optimistic: false # Faster, but does not perform DB compatibility checks
       }
     end
   end

data/lib/sequenceserver/job.rb

@@ -28,16 +28,25 @@ module SequenceServer
         enqueue(job)
       end
 
+      def serializable_classes
+        [
+          Time,
+          Symbol,
+          SequenceServer::Job,
+          SequenceServer::BLAST::Job,
+          SequenceServer::Database
+        ]
+      end
+
       # Fetches job with the given id.
       def fetch(id)
         job_file = File.join(DOTDIR, id, 'job.yaml')
-        fail NotFound unless File.exist?(job_file)
-        if RUBY_VERSION < '3.1.0'
-          YAML.load_file(job_file)
-        else
-          YAML.load_file(job_file, permitted_classes: [Time, Symbol, SequenceServer::BLAST::Job, \
-                                                       SequenceServer::Database, SequenceServer::Job])
-        end
+        return nil unless File.exist?(job_file)
+
+        YAML.safe_load_file(
+          job_file,
+          permitted_classes: serializable_classes
+        )
       end
 
       # Deletes job with the given id.
@@ -75,8 +84,8 @@ module SequenceServer
     # of job data will be held, yields (if block given) and saves the job.
     #
     # Subclasses should extend `initialize` as per requirement.
-    def initialize(*)
-      @id = SecureRandom.uuid
+    def initialize(params = {})
+      @id = params.fetch(:id, SecureRandom.uuid)
       @submitted_at = Time.now
       mkdir_p dir
       yield if block_given?
@@ -85,7 +94,7 @@ module SequenceServer
       raise SystemError, 'Not enough disk space to start a new job'
     rescue Errno::EACCES
       raise SystemError, "Permission denied to write to #{DOTDIR}"
-    rescue => e
+    rescue StandardError => e
       rm_rf dir
       raise e
     end
@@ -117,7 +126,7 @@ module SequenceServer
     # should be called on a completed job before attempting to use the results.
     # Subclasses should provide their own implementation.
     def raise!
-      raise if done? && exitstatus != 0
+      fail if done? && exitstatus != 0
     end
 
     # Where will the stdout be written to during execution and read from later.
@@ -163,6 +172,7 @@ module SequenceServer
     def fetch(key)
       filename = File.join(dir, key)
       fail unless File.exist? filename
+
       filename
     end