genevalidator 1.6.1 → 1.6.2

data/lib/genevalidator/sequences.rb

@@ -1,5 +1,7 @@
 require 'net/http'
 require 'io/console'
+require 'yaml'
+
 module GeneValidator
   # This is a class for the storing data on each sequence
   class Sequence
@@ -65,7 +67,7 @@ module GeneValidator
         end
         @raw_sequence = seq
       else
-        # puts "Tries to connect to the internet for #{accno}"
+        $stderr.puts "Getting sequence for '#{accno}' from NCBI - avoid this with '-r'."
         uri = 'http://www.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?'\
               "db=#{dbtype}&retmax=1&usehistory=y&term=#{accno}/"
         result = Net::HTTP.get(URI.parse(uri))
@@ -85,8 +87,6 @@ module GeneValidator
         @raw_sequence = '' unless @raw_sequence.index(/ERROR/).nil?
       end
       @raw_sequence
-      # rescue Exception => error
-      #   @raw_sequence = ""
     end
 
     ##

data/lib/genevalidator/tabular_parser.rb

@@ -1,5 +1,9 @@
-require 'genevalidator/exceptions'
 require 'csv'
+require 'forwardable'
+
+require 'genevalidator/sequences'
+require 'genevalidator/hsp'
+require 'genevalidator/exceptions'
 
 #
 module GeneValidator
@@ -8,6 +12,9 @@ module GeneValidator
   ##
   # This class parses the tabular output of BLAST (outfmt 6 & 7)
   class TabularParser
+    extend Forwardable
+    def_delegators GeneValidator, :opt, :config
+
     attr_reader :rows
     attr_reader :tab_results
     attr_reader :column_names
@@ -15,28 +22,25 @@ module GeneValidator
 
     ##
     # Initializes the object
-    # +file_content+ : String with the tabular BLAST output
-    # +format+: format of the tabular output (comma/space delimited string)
-    # +type+: :nucleotide or :mrna
-    def initialize(filename, format, type)
+    def initialize(tab_file = opt[:blast_tabular_file],
+                   format = opt[:blast_tabular_options], type = config[:type])
       @column_names = format.gsub(/[-\d]/, '').split(/[ ,]/)
-      @tab_results  = analayse_tabular_file(filename)
-      @rows         = @tab_results.to_enum
       @type         = type
+      @tab_results  = analayse_tabular_file(tab_file)
+      @rows         = @tab_results.to_enum
     end
 
     ##
     #
     def analayse_tabular_file(filename)
-      tab_results  = []
-      file         = File.read(filename)
-      lines        = CSV.parse(file, col_sep: "\t",
-                                     skip_lines: /^#/,
-                                     headers: @column_names)
+      results = []
+      file    = File.read(filename)
+      lines   = CSV.parse(file, col_sep: "\t", skip_lines: /^#/,
+                                headers: @column_names)
       lines.each do |line|
-        tab_results << line.to_hash
+        results << line.to_hash
       end
-      tab_results
+      results
     end
 
     ##
@@ -58,17 +62,19 @@ module GeneValidator
     def parse_next(query_id = nil)
       current_id = @rows.peek['qseqid']
       return [] if !query_id.nil? && current_id != query_id
-      hits = @tab_results.partition { |h| h['qseqid'] == current_id }[0]
-      hit_seq = initialise_classes(hits)
+      hit_seq = initialise_classes(current_id)
       move_to_next_query
       hit_seq
     rescue StopIteration
       return []
     end
 
+    private
+
     ##
     #
-    def initialise_classes(hits)
+    def initialise_classes(current_id, tab_results = @tab_results)
+      hits = tab_results.partition { |h| h['qseqid'] == current_id }[0]
       hit_list = []
       grouped_hits = hits.group_by { |row| row['sseqid'] }
 
@@ -90,7 +96,7 @@ module GeneValidator
       hsps = hits.select { |row| row['sseqid'] == current_query_id }
       hsps.each do |row|
         hsp = Hsp.new
-        hsp.init_tabular_attribute(row, type)
+        hsp.init_tabular_attribute(row)
         hit_seq.hsp_list.push(hsp)
       end
     end

data/lib/genevalidator/validation.rb

@@ -0,0 +1,279 @@
+require 'forwardable'
+
+require 'genevalidator/blast'
+require 'genevalidator/exceptions'
+require 'genevalidator/output'
+require 'genevalidator/pool'
+require 'genevalidator/sequences'
+require 'genevalidator/validation_length_cluster'
+require 'genevalidator/validation_length_rank'
+require 'genevalidator/validation_blast_reading_frame'
+require 'genevalidator/validation_gene_merge'
+require 'genevalidator/validation_duplication'
+require 'genevalidator/validation_open_reading_frame'
+require 'genevalidator/validation_alignment'
+
+# Top level module / namespace.
+module GeneValidator
+  Pair1 = Struct.new(:x, :y)
+
+  # Class that initalises separate Validate.new() instances for each query.
+  class Validations
+    extend Forwardable
+    def_delegators GeneValidator, :opt, :config, :query_idx
+    def initialize
+      @opt       = opt
+      @config    = config
+      @query_idx = query_idx
+    end
+
+    ##
+    #
+    def run_validations(iterator)
+      p = Pool.new(@opt[:num_threads]) if @opt[:num_threads] > 1
+
+      while @config[:idx] + 1 < @query_idx.length
+        prediction = get_info_on_query_sequence
+        @config[:idx] += 1
+
+        blast_hits = parse_next_iteration(iterator, prediction)
+
+        if blast_hits.nil?
+          @config[:idx] -= 1
+          break
+        end
+
+        if @opt[:num_threads] == 1
+          (Validate.new).validate(prediction, blast_hits, @config[:idx])
+        else
+          p.schedule(prediction, blast_hits, @config[:idx]) do |pred, hits, idx|
+            (Validate.new).validate(pred, hits, idx)
+          end
+        end
+      end
+    ensure
+      p.shutdown if @opt[:num_threads] > 1
+    end
+
+    ##
+    # get info about the query
+    def get_info_on_query_sequence(input_file = @opt[:input_fasta_file],
+                                        seq_type = @config[:type])
+      start_offset = @query_idx[@config[:idx] + 1] - @query_idx[@config[:idx]]
+      end_offset   = @query_idx[@config[:idx]]
+      query        = IO.binread(input_file, start_offset, end_offset)
+      parse_query  = query.scan(/>([^\n]*)\n([A-Za-z\n]*)/)[0]
+
+      prediction                = Sequence.new
+      prediction.definition     = parse_query[0].gsub("\n", '')
+      prediction.identifier     = prediction.definition.gsub(/ .*/, '')
+      prediction.type           = seq_type
+      prediction.raw_sequence   = parse_query[1].gsub("\n", '')
+      prediction.length_protein = prediction.raw_sequence.length
+      prediction.length_protein /= 3 if seq_type == :nucleotide
+      prediction
+    end
+
+    def parse_next_iteration(iterator, prediction)
+      iterator.next if @config[:idx] < @config[:start_idx]
+      if @opt[:blast_xml_file]
+        BlastUtils.parse_next(iterator)
+      elsif @opt[:blast_tabular_file]
+        iterator.parse_next(prediction.identifier)
+      end
+    end
+  end
+
+  # Class that runs the validations (Instatiated for each query)
+  class Validate
+    extend Forwardable
+    def_delegators GeneValidator, :opt, :config, :mutex_array, :overview
+
+    ##
+    # Initilizes the object
+    # Params:
+    # +opt+: A hash with the following keys: validations:, blast_tabular_file:,
+    # blast_tabular_options:, blast_xml_file:, db:, raw_sequences:,
+    # num_threads:, fast:}
+    # +start_idx+: number of the sequence from the file to start with
+    # +overall_evaluation+: boolean variable for printing overall evaluation
+    def initialize
+      @opt         = opt
+      @config      = config
+      @mutex_array = mutex_array
+      @run_output  = nil
+
+      @overview    = overview
+    end
+
+    ##
+    # Validate one query and create validation report
+    # Params:
+    # +prediction+: Sequence object
+    # +hits+: Array of +Sequence+ objects
+    # +current_idx+: the index number of the query
+    def validate(prediction, hits, current_idx)
+      hits = remove_identical_hits(prediction, hits)
+      vals = create_validation_tests(prediction, hits)
+      check_validations(vals)
+      vals.each(&:run)
+      @run_output = Output.new(current_idx, hits.length, prediction.definition)
+      @run_output.validations = vals.map(&:validation_report)
+      check_validations_output(vals)
+
+      compute_scores
+      generate_run_output
+    end
+
+    ##
+    # Removes identical hits (100% coverage and >99% identity)
+    # Params:
+    # +prediction+: Sequence object
+    # +hits+: Array of +Sequence+ objects
+    # Output:
+    # new array of hit +Sequence+ objects
+    def remove_identical_hits(prediction, hits)
+      identical_hits = []
+      hits.each do |hit|
+        low_identity = hit.hsp_list.select { |hsp| hsp.pidentity < 99 }
+        no_data      = hit.hsp_list.select { |hsp| hsp.pidentity.nil? }
+        low_identity += no_data
+        # check the coverage
+        coverage = Array.new(prediction.length_protein, 0)
+        hit.hsp_list.each do |hsp|
+          match_to   = hsp.match_query_to
+          match_from = hsp.match_query_from
+          len        = match_to - match_from + 1
+          coverage[match_from - 1..match_to - 1] = Array.new(len, 1)
+        end
+
+        if low_identity.length == 0 && coverage.uniq.length == 1
+          identical_hits.push(hit)
+        end
+      end
+
+      identical_hits.each { |hit| hits.delete(hit) }
+      hits
+    end
+
+    def create_validation_tests(prediction, hits)
+      val = []
+      val.push LengthClusterValidation.new(prediction, hits)
+      val.push LengthRankValidation.new(prediction, hits)
+      val.push GeneMergeValidation.new(prediction, hits)
+      val.push DuplicationValidation.new(prediction, hits)
+      if @config[:type] == :nucleotide
+        val.push BlastReadingFrameValidation.new(prediction, hits)
+        val.push OpenReadingFrameValidation.new(prediction, hits)
+      end
+      val.push AlignmentValidation.new(prediction, hits)
+      val.select { |v| @opt[:validations].include? v.cli_name.downcase }
+    end
+
+    def check_validations(vals)
+      # check the class type of the elements in the list
+      vals.each { |v| fail ValidationClassError unless v.is_a? ValidationTest }
+      # check alias duplication
+      aliases = vals.map(&:cli_name)
+      fail AliasDuplicationError unless aliases.length == aliases.uniq.length
+    rescue ValidationClassError => e
+      $stderr.puts e
+      exit 1
+    rescue AliasDuplicationError => e
+      $stderr.puts e
+      exit 1
+    end
+
+    def check_validations_output(vals)
+      fail NoValidationError if @run_output.validations.length == 0
+      vals.each do |v|
+        fail ReportClassError unless v.validation_report.is_a? ValidationReport
+      end
+    rescue NoValidationError => e
+      $stderr.puts e
+      exit 1
+    rescue ReportClassError => e
+      $stderr.puts e
+      exit 1
+    end
+
+    def compute_scores
+      validations        = @run_output.validations
+      scores             = {}
+      scores[:successes] = validations.map { |v| v.result == v.expected }.count(true)
+      scores[:fails] = validations.map { |v| v.validation != :unapplicable && v.validation != :error && v.result != v.expected }.count(true)
+      scores         = length_validation_scores(validations, scores)
+
+      @run_output.successes     = scores[:successes]
+      @run_output.fails         = scores[:fails]
+      total_query              = scores[:successes].to_i + scores[:fails]
+      @run_output.overall_score = (scores[:successes] * 100 / total_query).round
+    end
+
+    # Since there are two length validations, it is necessary to adjust the
+    #   scores accordingly
+    def length_validation_scores(validations, scores)
+      lcv = validations.select { |v| v.class == LengthClusterValidationOutput }
+      lrv = validations.select { |v| v.class == LengthRankValidationOutput }
+      if lcv.length == 1 && lrv.length == 1
+        score_lcv = (lcv[0].result == lcv[0].expected)
+        score_lrv = (lrv[0].result == lrv[0].expected)
+        if score_lcv == true && score_lrv == true
+          scores[:successes] -= 1 # if both are true: counted as 1 success
+        elsif score_lcv == false && score_lrv == false
+          scores[:fails] -= 1 # if both are false: counted as 1 fail
+        else
+          scores[:successes] -= 0.5
+          scores[:fails] -= 0.5
+        end
+      end
+      scores
+    end
+
+    def generate_run_output
+      @run_output.generate_html
+      @run_output.generate_json
+      @run_output.print_output_console
+      generate_run_overview
+    end
+
+    def generate_run_overview
+      vals        = @run_output.validations
+      no_mafft    = 0
+      no_internet = 0
+      errors      = []
+      vals.each do |v|
+        unless v.errors.nil?
+          no_mafft += v.errors.select { |e| e == NoMafftInstallationError }.length
+          no_internet += v.errors.select { |e| e == NoInternetError }.length
+        end
+        errors.push(v.short_header) if v.validation == :error
+      end
+
+      no_evidence = vals.count { |v| v.result == :unapplicable || v.result == :warning } == vals.length
+      nee = (no_evidence) ? 1 : 0
+
+      good_scores = (@run_output.overall_score >= 75) ? 1 : 0
+      bad_scores  = (@run_output.overall_score >= 75) ? 0 : 1
+
+      @mutex_array.synchronize do
+        @overview[:no_queries] += 1
+        @overview[:scores].push(@run_output.overall_score)
+        @overview[:good_scores] += good_scores
+        @overview[:bad_scores] += bad_scores
+        @overview[:nee] += nee
+        @overview[:no_mafft] += no_mafft
+        @overview[:no_internet] += no_internet
+        errors.each { |err| @overview[:map_errors][err] += 1 }
+
+        vals.each do |v|
+          next if v.run_time == 0 || v.run_time.nil?
+          next if v.validation == :unapplicable || v.validation == :error
+          p = Pair1.new(@overview[:run_time][v.short_header].x + v.run_time,
+                        @overview[:run_time][v.short_header].y + 1)
+          @overview[:run_time][v.short_header] = p
+        end
+      end
+    end
+  end
+end

data/lib/genevalidator/validation_alignment.rb

@@ -1,5 +1,9 @@
-require 'genevalidator/validation_report'
+require 'bio'
+require 'forwardable'
+
 require 'genevalidator/exceptions'
+require 'genevalidator/validation_report'
+require 'genevalidator/validation_test'
 
 module GeneValidator
   ##
@@ -81,7 +85,8 @@ module GeneValidator
   # This class contains the methods necessary for
   # validations based on multiple alignment
   class AlignmentValidation < ValidationTest
-    attr_reader :filename
+    extend Forwardable
+    def_delegators GeneValidator, :opt, :config
     attr_reader :multiple_alignment
     attr_reader :raw_seq_file
     attr_reader :index_file_name
@@ -90,32 +95,26 @@ module GeneValidator
     ##
     # Initilizes the object
     # Params:
-    # +type+: type of the predicted sequence (:nucleotide or :protein)
     # +prediction+: a +Sequence+ object representing the blast query
     # +hits+: a vector of +Sequence+ objects (representing blast hits)
-    # +filename+: name of the fasta file
-    # +mafft_path+: path of the MAFFT installation
-    # +raw_seq_file+: name of the fasta file with raw sequences
-    # +index_file_name+: name of the fasta index file
-    # +raw_seq_file_load+: String - loaded content of the index file
-    def initialize(type, prediction, hits, filename, raw_seq_file,
-                   index_file_name, raw_seq_file_load, db, num_threads)
+    # +plot_path+: name of the fasta file
+    def initialize(prediction, hits)
       super
-      @short_header       = 'MA'
-      @header             = 'Missing/Extra sequences'
+      @short_header       = 'MissingExtraSequences'
+      @cli_name           = 'align'
+      @header             = 'Missing/Extra Sequences'
       @description        = 'Finds missing and extra sequences in the' \
                             ' prediction, based on the multiple alignment of' \
                             ' the best hits. Also counts the percentage of' \
                             ' the conserved regions that appear in the' \
                             ' prediction.'
-      @filename           = filename
-      @raw_seq_file       = raw_seq_file
-      @index_file_name    = index_file_name
-      @raw_seq_file_load  = raw_seq_file_load
-      @db                 = db
+      @raw_seq_file       = opt[:raw_sequences]
+      @index_file_name    = config[:raw_seq_file_index]
+      @raw_seq_file_load  = config[:raw_seq_file_load]
+      @db                 = opt[:db]
       @multiple_alignment = []
-      @cli_name           = 'align'
-      @num_threads        = num_threads
+      @num_threads        = opt[:num_threads]
+      @type               = config[:type]
     end
 
     ##
@@ -189,39 +188,32 @@ module GeneValidator
                                                          @description, gaps,
                                                          extra_seq, consensus)
       @validation_report.plot_files.push(plot1)
-      @validation_report.running_time = Time.now - start
+      @validation_report.run_time = Time.now - start
       @validation_report
 
     rescue NotEnoughHitsError
       @validation_report = ValidationReport.new('Not enough evidence',
                                                 :warning, @short_header,
-                                                @header, @description,
-                                                @approach, @explanation,
-                                                @conclusion)
+                                                @header, @description)
     rescue NoMafftInstallationError
       @validation_report = ValidationReport.new('Mafft error', :error,
                                                 @short_header, @header,
-                                                @description, @approach,
-                                                @explanation, @conclusion)
+                                                @description)
       @validation_report.errors.push NoMafftInstallationError
     rescue NoInternetError
       @validation_report = ValidationReport.new('Internet error', :error,
                                                 @short_header, @header,
-                                                @description, @approach,
-                                                @explanation, @conclusion)
+                                                @description)
       @validation_report.errors.push NoInternetError
     rescue ReadingFrameError
       @validation_report = ValidationReport.new('Multiple reading frames',
                                                 :error, @short_header,
-                                                @header, @description,
-                                                @approach, @explanation,
-                                                @conclusion)
+                                                @header, @description)
       @validation_report.errors.push 'Multiple reading frames Error'
     rescue Exception
       @validation_report = ValidationReport.new('Unexpected error', :error,
                                                 @short_header, @header,
-                                                @description, @approach,
-                                                @explanation, @conclusion)
+                                                @description)
       @validation_report.errors.push 'Unexpected Error'
     end
 
@@ -417,9 +409,9 @@ module GeneValidator
     # lines for multiple hits alignment, prediction and statistical model
     # Params:
     # +freq+: +String+ residue frequency from the statistical model
-    # +output+: filename of the json file
+    # +output+: plot_path of the json file
     # +ma+: +String+ array with the multiple alignmened hits and prediction
-    def plot_alignment(freq, output = "#{@filename}_ma.json", ma = @multiple_alignment)
+    def plot_alignment(freq, ma = @multiple_alignment)
       # get indeces of consensus in the multiple alignment
       consensus = get_consensus(@multiple_alignment[0..@multiple_alignment.length - 2])
       consensus_idxs = consensus.split(//).each_index.select { |j| isalpha(consensus[j]) }
@@ -438,32 +430,24 @@ module GeneValidator
 
       len = ma[0].length
 
-      f = File.open(output, 'w')
-      f.write((
       # plot statistical model
-      freq.each_with_index.map { |f, j| { 'y' => ma.length, 'start' => j, 'stop' => j + 1, 'color' => 'orange', 'height' => f } } +
+      data = freq.each_with_index.map { |h, j| { 'y' => ma.length, 'start' => j, 'stop' => j + 1, 'color' => 'orange', 'height' => h } } +
       # hits
       match_alignment_ranges.each_with_index.map { |ranges, j| ranges.map { |range| { 'y' => ma.length - j - 1, 'start' => range.first, 'stop' => range.last, 'color' => 'red', 'height' => -1 } } }.flatten +
-      ma[0..ma.length - 2].each_with_index.map { |_seq, j|
-        consensus_ranges.map { |range| { 'y' => j + 1, 'start' => range.first, 'stop' => range.last, 'color' => 'yellow', 'height' => -1 } }
-      }.flatten +
+      ma[0..ma.length - 2].each_with_index.map { |_seq, j| consensus_ranges.map { |range| { 'y' => j + 1, 'start' => range.first, 'stop' => range.last, 'color' => 'yellow', 'height' => -1 } } }.flatten +
       # plot prediction
       [{ 'y' => 0, 'start' => 0, 'stop' => len, 'color' => 'gray', 'height' => -1 }] +
       query_alignment_ranges.map { |range| { 'y' => 0, 'start' => range.first, 'stop' => range.last, 'color' => 'red', 'height' => -1 } }.flatten +
 
       # plot consensus
-      consensus_all_ranges.map { |range| { 'y' => 0, 'start' => range.first, 'stop' => range.last, 'color' => 'yellow', 'height' => -1 } }.flatten).to_json)
-
-      f.close
+      consensus_all_ranges.map { |range| { 'y' => 0, 'start' => range.first, 'stop' => range.last, 'color' => 'yellow', 'height' => -1 } }.flatten
 
       yAxisValues = 'Prediction'
-      (1..ma.length - 1).each do |i|
-        yAxisValues << ", hit&nbsp;#{i}"
-      end
+      (1..ma.length - 1).each { |i| yAxisValues << ", hit #{i}" }
 
       yAxisValues << ', Statistical Model'
 
-      Plot.new(output.scan(%r{([^/]+)$})[0][0],
+      Plot.new(data,
                :align,
                'Missing/Extra sequences Validation: Multiple Align. & Statistical model of hits',
                'Conserved Region, Yellow',