genevalidator 1.6.12 → 2.1.3

data/lib/genevalidator/exceptions.rb

@@ -1,11 +1,11 @@
 module GeneValidator
   # Exception raised when BLAST path is not added to the CLASSPATH
-  class ClasspathError < Exception
+  class ClasspathError < RuntimeError
   end
 
   # Exception raised when the command line type argument
   # does not corrsepond to the type of the sequences in the fasta file
-  class SequenceTypeError < Exception
+  class SequenceTypeError < RuntimeError
     def to_s
       "\nSequence Type error: Possible cause include that the blast output" \
       " was not obtained against a protein database.\n"
@@ -13,15 +13,15 @@ module GeneValidator
   end
 
   # Exception raised when an unexisting file is accessed
-  class FileNotFoundException < Exception
+  class FileNotFoundException < RuntimeError
   end
 
   # Exception raised when blast does not find any hit
-  class QueryError < Exception
+  class QueryError < RuntimeError
   end
 
   # Exception raised when a validation class is not instance of ValidationTest
-  class ValidationClassError < Exception
+  class ValidationClassError < RuntimeError
     def to_s
       "\nClass Type error: Possible cause include that one of the validations" \
       " is not a sub-class of ValidationTest\n"
@@ -30,7 +30,7 @@ module GeneValidator
 
   # Exception raised when a validation report class is not instance of
   #   ValidationReport
-  class ReportClassError < Exception
+  class ReportClassError < RuntimeError
     def to_s
       "\nClass Type error: Possible causes include that the type of one of" \
       ' the validation reports is not a subclass of the ValidationReport' \
@@ -40,20 +40,20 @@ module GeneValidator
 
   # Exception raised when there are not enough blast hits to make a statistical
   #   validation
-  class NotEnoughHitsError < Exception
+  class NotEnoughHitsError < RuntimeError
   end
 
   # Exception raised when function dependig on the internet connection raise
   #   Exception
-  class NoInternetError < Exception
+  class NoInternetError < RuntimeError
   end
 
   # Exception raised when the alignment initialization raises exception
-  class NoMafftInstallationError < Exception
+  class NoMafftInstallationError < RuntimeError
   end
 
   # Exception raised when the -v argument didn't filter any validatio test
-  class NoValidationError < Exception
+  class NoValidationError < RuntimeError
     def to_s
       "\nValidation error: Possible cause inlcude that the -v arguments" \
       " supplied is not valid\n"
@@ -61,7 +61,7 @@ module GeneValidator
   end
 
   # Exception raised when the are alias duplications
-  class AliasDuplicationError < Exception
+  class AliasDuplicationError < RuntimeError
     def to_s
       "\nAlias Duplication error: Possible cause: At least two validations" \
       " have the same CLI alias\n"
@@ -69,27 +69,27 @@ module GeneValidator
   end
 
   # Exception raised when the BLAST is not set up with the '-parse-seqids' arg.
-  class BLASTDBError < Exception
+  class BLASTDBError < RuntimeError
   end
 
   # Error raised by QI Validation when the query does not have QI tag
-  class NotEnoughEvidence < Exception
+  class NotEnoughEvidence < RuntimeError
   end
 
   # Exception raised when the are alias duplications
-  class NoPIdentError < Exception
+  class NoPIdentError < RuntimeError
   end
 
   # Exception raised when the tabular format does not correspond to the tabular
   #   argument
-  class InconsistentTabularFormat < Exception
+  class InconsistentTabularFormat < RuntimeError
   end
 
   # Exception raised when there are more than one reading frame among the hits
   #   of one prediction
-  class ReadingFrameError < Exception
+  class ReadingFrameError < RuntimeError
   end
 
-  class OtherError < Exception
+  class OtherError < RuntimeError
   end
 end

data/lib/genevalidator/ext/array.rb

@@ -9,15 +9,15 @@ module GeneValidator
       sum / length.to_f
     end
 
-    def median
-      sorted = sort
+    def median(already_sorted = false)
+      sorted = already_sorted ? self : sort
       len    = sorted.length
       (sorted[(len - 1) / 2] + sorted[len / 2]) / 2.0
     end
 
     def mode
-      freq = inject(Hash.new(0)) { |h, v| h[v] += 1; h }
-      sort_by { |v| freq[v] }.last
+      freq = each_with_object(Hash.new(0)) { |v, h| h[v] += 1; }
+      max_by { |v| freq[v] }
     end
 
     def sample_variance
@@ -29,6 +29,23 @@ module GeneValidator
     def standard_deviation
       Math.sqrt(sample_variance)
     end
+
+    def all_quartiles
+      sorted = sort
+      len    = sorted.length
+      split  = sorted.median_split
+      [
+        split[0].median(true),
+        sorted.median(true),
+        split[1].median(true)
+      ]
+    end
+
+    def median_split
+      len    = length
+      center = len % 2
+      [self[0..len / 2 - 1], self[len / 2 + center..-1]]
+    end
   end
 end
 

data/lib/genevalidator/get_raw_sequences.rb

@@ -13,24 +13,25 @@ module GeneValidator
   class RawSequences
     class <<self
       extend Forwardable
-      def_delegators GeneValidator, :opt, :config
+      def_delegators GeneValidator, :opt, :config, :dirs
 
       def init
-        $stderr.puts 'Extracting sequences within the BLAST output file from' \
-                     ' the BLAST database'
+        warn '==> Extracting fasta sequences for each BLAST HSP from the' \
+             ' BLAST database'
 
         @blast_file = opt[:blast_xml_file] if opt[:blast_xml_file]
         @blast_file = opt[:blast_tabular_file] if opt[:blast_tabular_file]
 
-        opt[:raw_sequences] = @blast_file + '.raw_seq'
-        @index_file         = @blast_file + '.index'
+        fname = File.basename(@blast_file)
+        opt[:raw_sequences] = File.join(dirs[:tmp_dir], "#{fname}.raw_seq")
+        @index_file         = File.join(dirs[:tmp_dir], "#{fname}.index")
       end
 
       ##
       # Obtains raw_sequences from BLAST output file...
       def run
         init
-        if opt[:db] =~ /remote/
+        if opt[:db].match?(/remote/)
           write_a_raw_seq_file(opt[:raw_sequences], 'remote')
         else
           write_an_index_file(@index_file, 'local')
@@ -55,12 +56,13 @@ module GeneValidator
         index_hash = {}
         keys.each_with_index do |k, i|
           start = values[i]
-          endf  = (i == values.length - 1) ? content.length - 1 : values[i + 1]
+          endf  = i == values.length - 1 ? content.length - 1 : values[i + 1]
           index_hash[k] = [start, endf]
         end
 
         # create FASTA index
-        config[:raw_seq_file_index] = "#{raw_seq_file}.idx"
+        fname = File.basename(raw_seq_file)
+        config[:raw_seq_file_index] = File.join(dirs[:tmp_dir], "#{fname}.idx")
         config[:raw_seq_file_load]  = index_hash
 
         File.open(config[:raw_seq_file_index], 'w') do |f|
@@ -76,29 +78,29 @@ module GeneValidator
         iterate_xml(file, db_type) if opt[:blast_xml_file]
         iterate_tabular(file, db_type) if opt[:blast_tabular_file]
       rescue BLASTDBError
-        $stderr.puts "*** BLAST Database Error: Genevalidator requires BLAST" \
+        warn '*** BLAST Database Error: Genevalidator requires BLAST' \
         " databases to be created with the '-parse_seqids argument."
-        $stderr.puts "    See https://github.com/wurmlab/genevalidator" \
-        "#setting-up-a-blast-database for more information"
+        warn '    See https://github.com/wurmlab/genevalidator' \
+        '#setting-up-a-blast-database for more information'
         exit 1
-      rescue
-        $stderr.puts '*** Error: There was an error in analysing the BLAST'
-        $stderr.puts '    output file. Please ensure that BLAST output file'
-        $stderr.puts '    is in the correct format and then try again. If you'
-        $stderr.puts '    are using a remote database, please ensure that you'
-        $stderr.puts '    have internet access.'
+      rescue StandardError
+        warn '*** Error: There was an error in analysing the BLAST'
+        warn '    output file. Please ensure that BLAST output file'
+        warn '    is in the correct format and then try again. If you'
+        warn '    are using a remote database, please ensure that you'
+        warn '    have internet access.'
         exit 1
       ensure
         file.close unless file.nil?
       end
 
-      alias_method :write_a_raw_seq_file, :write_an_index_file
+      alias write_a_raw_seq_file write_an_index_file
 
       def iterate_xml(file, db_type)
         n = Bio::BlastXMLParser::XmlIterator.new(opt[:blast_xml_file]).to_enum
         n.each do |iter|
           iter.each do |hit|
-            fail BLASTDBError if hit.hit_id =~ /\|BL_ORD_ID\|/
+            raise BLASTDBError if hit.hit_id =~ /\|BL_ORD_ID\|/
             if db_type == 'remote' || hit.hit_id.nil?
               file.puts FetchRawSequences.extract_from_remote_db(hit.accession)
             else
@@ -116,7 +118,7 @@ module GeneValidator
                                    headers: table_headers)
 
         rows.each do |row|
-          fail BLASTDBError if row['sseqid'] =~ /\|BL_ORD_ID\|/
+          raise BLASTDBError if row['sseqid'] =~ /\|BL_ORD_ID\|/i
           if db_type == 'remote' || row['sseqid'].nil?
             file.puts FetchRawSequences.extract_from_remote_db(row['sacc'])
           else
@@ -136,15 +138,15 @@ module GeneValidator
         # first try to extract from previously created raw_sequences HASH
         raw_seq = extract_from_index(identifier) if opt[:raw_sequences]
         # then try to just extract that sequence based on accession.
-        if opt[:db] !~ /remote/ && (raw_seq.nil? || raw_seq =~ /Error/)
+        if opt[:db] !~ /remote/ && (raw_seq.nil? || raw_seq =~ /Error/i)
           raw_seq = extract_from_local_db(false, accession)
         end
         # then try to extract from remote database
-        if opt[:db] =~ /remote/ && (raw_seq.nil? || raw_seq =~ /Error/)
+        if opt[:db] =~ /remote/ && (raw_seq.nil? || raw_seq =~ /Error/i)
           raw_seq = extract_from_remote_db(accession)
         end
         # return nil if the raw_sequence still produces an error.
-        (raw_seq =~ /Error/) ? nil : raw_seq
+        raw_seq =~ /Error/i ? nil : raw_seq
       end
 
       ##
@@ -157,8 +159,8 @@ module GeneValidator
         idx         = config[:raw_seq_file_load][identifier]
         query       = IO.binread(opt[:raw_sequences], idx[1] - idx[0], idx[0])
         parse_query = query.scan(/>([^\n]*)\n([A-Za-z\n]*)/)[0]
-        parse_query[1].gsub("\n", '')
-      rescue
+        parse_query[1].delete("\n")
+      rescue StandardError
         'Error' # return error so it can then try alternative fetching method.
       end
 
@@ -170,7 +172,7 @@ module GeneValidator
       # Output:
       # String with the nucleotide sequence corresponding to the accession
       def extract_from_local_db(batch, accno = nil, idx_file = nil)
-        cmd = (batch) ? batch_raw_seq_cmd(idx_file) : single_raw_seq_cmd(accno)
+        cmd = batch ? batch_raw_seq_cmd(idx_file) : single_raw_seq_cmd(accno)
         efile = Tempfile.new('blast_out')
         `#{cmd} &>#{efile.path}`
         raw_seqs = efile.read
@@ -193,9 +195,9 @@ module GeneValidator
       def failed_raw_sequences(blast_output)
         blast_output.each_line do |line|
           acc = line.match(/Error: (\w+): OID not found/)[1]
-          $stderr.puts "\nCould not find sequence '#{acc.chomp}' within the" \
+          warn "\nCould not find sequence '#{acc.chomp}' within the" \
                        ' BLAST database.'
-          $stderr.puts "Attempting to obtain sequence '#{acc.chomp}' from" \
+          warn "Attempting to obtain sequence '#{acc.chomp}' from" \
                        ' remote BLAST databases.'
           File.open(opt[:raw_sequences], 'a+') do |f|
             f.puts extract_from_remote_db(acc)
@@ -204,13 +206,12 @@ module GeneValidator
       end
 
       def extract_from_remote_db(accession, db_seq_type = 'protein')
-        uri     = 'http://www.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?' \
+        uri     = 'https://www.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?' \
                   "db=#{db_seq_type}&retmax=1&usehistory=y&term=#{accession}/"
         result  = Net::HTTP.get(URI.parse(uri))
         query   = result.match(%r{<\bQueryKey\b>([\w\W\d]+)</\bQueryKey\b>})[1]
         web_env = result.match(%r{<\bWebEnv\b>([\w\W\d]+)</\bWebEnv\b>})[1]
-
-        uri     = 'http://www.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?' \
+        uri     = 'https://www.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?' \
                   'rettype=fasta&retmode=text&retstart=0&retmax=1&' \
                   "db=#{db_seq_type}&query_key=#{query}&WebEnv=#{web_env}"
         result  = Net::HTTP.get(URI.parse(uri))

data/lib/genevalidator/hsp.rb

@@ -1,9 +1,14 @@
+require 'forwardable'
+
 require 'genevalidator/blast'
 require 'genevalidator/exceptions'
 
 module GeneValidator
   # A class that initialises the BLAST tabular attributes
   class Hsp
+    extend Forwardable
+    def_delegators GeneValidator, :config
+
     attr_accessor :hit_from # ref. from the unaligned hit sequence
     attr_accessor :hit_to
     attr_accessor :match_query_from # ref. from the unaligned query sequence
@@ -23,9 +28,30 @@ module GeneValidator
     attr_accessor :gaps
     attr_accessor :align_len
 
-    def initialize
+    def initialize(input = {})
       @query_alignment = nil
       @hit_alignment   = nil
+      init_xml_attributes(input[:xml_input]) if input[:xml_input]
+      init_tabular_attribute(input[:tabular_input]) if input[:tabular_input]
+    end
+
+    def init_xml_attributes(hsp)
+      @match_query_from    = hsp.query_from.to_i
+      @match_query_to      = hsp.query_to.to_i
+      @query_reading_frame = hsp.query_frame.to_i
+      @hit_from            = hsp.hit_from.to_i
+      @hit_to              = hsp.hit_to.to_i
+      @query_alignment     = hsp.qseq.to_s
+      @hit_alignment       = hsp.hseq.to_s
+      @align_len           = hsp.align_len.to_i
+      @pidentity           = (100 * hsp.identity / hsp.align_len.to_f).round(2)
+      @identity            = hsp.identity.to_i
+      @hsp_evalue          = format('%.0e', hsp.evalue)
+      assert_seq_type(@hit_alignment) if @hit_alignment
+      assert_seq_type(@query_alignment) if @query_alignment
+      return unless config[:type] == :nucleotide
+      @match_query_from = (@match_query_from / 3) + 1
+      @match_query_to   = (@match_query_to / 3) + 1
     end
 
     ##
@@ -52,7 +78,10 @@ module GeneValidator
 
     def assert_seq_type(query)
       seq_type = BlastUtils.guess_sequence_type(query)
-      fail SequenceTypeError if seq_type != :protein
+      raise SequenceTypeError if seq_type != :protein
+    rescue SequenceTypeError => e
+      warn e
+      exit 1
     end
   end
 end

data/lib/genevalidator/json_to_gv_results.rb

@@ -1,10 +1,7 @@
-require 'erb'
-require 'fileutils'
 require 'forwardable'
 require 'json'
 
 require 'genevalidator'
-require 'genevalidator/output'
 require 'genevalidator/version'
 
 module GeneValidator
@@ -12,139 +9,58 @@ module GeneValidator
   class JsonToGVResults
     class << self
       extend Forwardable
-      def_delegators GeneValidator, :opt
+      def_delegators GeneValidator, :opt, :config, :dirs
 
-      def init
-        @opt    = opt
-        @config = {
-          html_path: "#{@opt[:json_file]}.html",
-          plot_dir: "#{@opt[:json_file]}.html/files/json",
-          aux: File.expand_path(File.join(File.dirname(__FILE__), '../../aux')),
-          filename: File.basename(@opt[:json_file]),
-          output_max: 2500,
-          run_no: 0
-        }
-        @json_array = load_json_file
+      def init(opt)
+        GeneValidator.opt = opt
+        GeneValidator.config = { output_max: 2500, run_no: 0,
+                                 json_output: load_json_file }
+        GeneValidator.dirs = GeneValidator.setup_dirnames(opt[:json_file])
       end
 
       def run
-        init
-        GeneValidator.create_output_folder(@config[:html_path], @config[:aux])
-        @json_array.each do |row|
-          @config[:run_no] += 1
-          create_json_file(row)
-          output_html = output_filename
-          generate_html_header(output_html) unless File.exist?(output_html)
-          generate_html_query(output_html, row)
+        warn '==> Parsing input JSON results'
+        print_console_header(config[:json_output][0])
+        config[:json_output].each do |row|
+          print_output_console(row)
+          create_row_json_plot_files(row)
         end
-        html_footer
-        calculate_overall_score
+        GeneValidator.produce_output
       end
 
-      def load_json_file
-        json_contents = File.read(File.expand_path(@opt[:json_file]))
-        JSON.load(json_contents)
+      def print_console_header(first_row)
+        return unless opt[:output_formats].include? 'stdout'
+        return if config[:console_header_printed]
+        config[:console_header_printed] = true
+        warn '' # blank line
+        c_fmt = "%3s\t%5s\t%20s\t%7s\t"
+        print format(c_fmt, 'No', 'Score', 'Identifier', 'No_Hits')
+        puts first_row[:validations].keys.join("\t")
       end
 
-      def create_json_file(row)
-        @json_file = File.join(@config[:plot_dir],
-                               "#{@config[:filename]}_#{row['idx']}.json")
-        File.open(@json_file, 'w') { |f| f.write(row.to_json) }
-      end
-
-      def output_filename
-        i = (@config[:run_no].to_f / @config[:output_max]).ceil
-        File.join(@config[:html_path], "results#{i}.html")
-      end
-
-      def generate_html_header(output_html)
-        return if File.exist?(output_html)
-        json_header_template = File.join(@config[:aux], 'json_header.erb')
-        template_contents    = File.open(json_header_template, 'r').read
-        erb                  = ERB.new(template_contents, 0, '>')
-        File.open(output_html, 'w+') { |f| f.write(erb.result(binding)) }
-      end
-
-      def generate_html_query(output_html, row)
-        @row = row
-        json_query_template = File.join(@config[:aux], 'json_query.erb')
-        template_contents   = File.open(json_query_template, 'r').read
-        erb                 = ERB.new(template_contents, 0, '>')
-        File.open(output_html, 'a') { |f| f.write(erb.result(binding)) }
-      end
-
-      # Add footer to all output files
-      def html_footer
-        no_of_output_files = (@config[:run_no].to_f / @config[:output_max]).ceil
-
-        output_files = []
-        (1..no_of_output_files).each { |i| output_files << "results#{i}.html" }
-
-        write_html_footer(no_of_output_files, output_files)
-      end
-
-      def write_html_footer(no_of_output_files, output_files)
-        turn_off_automated_sorting
-        json_footer_template = File.join(@config[:aux], 'json_footer.erb')
-        template_contents    = File.open(json_footer_template, 'r').read
-        erb                  = ERB.new(template_contents, 0, '>')
-        (1..no_of_output_files).each do |i|
-          results_html = File.join(@config[:html_path], "results#{i}.html")
-          File.open(results_html, 'a+') { |f| f.write(erb.result(binding)) }
+      def print_output_console(row)
+        return unless opt[:output_formats].include? 'stdout'
+        c_fmt = "%3s\t%5s\t%20s\t%7s\t"
+        short_def = row[:definition].split(' ')[0]
+        print format(c_fmt, row[:idx], row[:overall_score], short_def,
+          row[:no_hits])
+          puts row[:validations].values.map { |e| e[:print] }.join("\t")
+          .gsub('&nbsp;', ' ')
         end
-      end
-
-      # By default, on page load, the results are automatically sorted by the
-      # index. However since the whole idea is that users would sort by JSON,
-      # this is not wanted here.
-      def turn_off_automated_sorting
-        script_file = File.join(@config[:html_path],
-                                'files/js/genevalidator.compiled.min.js')
-        original_content = File.read(script_file)
-        # removes the automatic sort on page load
-        updated_content = original_content.gsub(',sortList:[[0,0]]', '')
-        File.open("#{script_file}.tmp", 'w') { |f| f.puts updated_content }
-        FileUtils.mv("#{script_file}.tmp", script_file)
-      end
 
-      def calculate_overall_score
-        scores = []
-        @json_array.each { |row| scores << row['overall_score'] }
-        plot_dir = File.join(@config[:html_path], 'files/json')
-        less     =  generate_evaluation(scores)
-        Output.create_overview_json(scores, plot_dir, less, less)
-      end
-
-      def generate_evaluation(scores)
-        no_of_queries = scores.length
-        good_scores = scores.count { |s| s >= 75 }
-        bad_scores  = scores.count { |s| s < 75 }
-        nee         = calculate_no_quries_with_no_evidence # nee = no evidence
+        private
 
-        good_pred = (good_scores == 1) ? 'One' : "#{good_scores} are"
-        bad_pred  = (bad_scores == 1) ? 'One' : "#{bad_scores} are"
-        eval = 'Overall Query Score Evaluation:<br>' \
-               "#{no_of_queries} predictions were validated, from which there" \
-               ' were:<br>' \
-               "#{good_pred} good prediction(s),<br>" \
-               "#{bad_pred} possibly weak prediction(s).<br>"
-        return eval if nee == 0
-        eval << "#{nee} could not be evaluated due to the lack of" \
-                ' evidence.<br>'
-        eval
-      end
+        def load_json_file
+          json_contents = File.read(File.expand_path(opt[:json_file]))
+          JSON.parse(json_contents, symbolize_names: true)
+        end
 
-      # calculate number of queries that had warnings for all validations.
-      def calculate_no_quries_with_no_evidence
-        all_warnings = 0
-        @json_array.each do |row|
-          status = row['validations'].map { |_, h| h['status'] }
-          if status.count { |r| r == 'warning' } == status.length
-            all_warnings += 1
-          end
+        def create_row_json_plot_files(row)
+          config[:run_no] += 1
+          fname = "#{dirs[:filename]}_#{row[:idx]}.json"
+          json_file = File.join(dirs[:json_dir], fname)
+          File.open(json_file, 'w') { |f| f.write(row.to_json) }
         end
-        all_warnings
-      end
     end
   end
 end