genevalidator 1.6.2 → 1.6.3

data/lib/genevalidator/exceptions.rb

@@ -3,7 +3,6 @@ module GeneValidator
   class ClasspathError < Exception
   end
 
-
   # Exception raised when the command line type argument
   # does not corrsepond to the type of the sequences in the fasta file
   class SequenceTypeError < Exception

data/lib/genevalidator/get_raw_sequences.rb

@@ -1,11 +1,12 @@
 require 'bio-blastxmlparser'
 require 'forwardable'
 require 'net/http'
+require 'tempfile'
 require 'uri'
 require 'yaml'
 
 require 'genevalidator/exceptions'
-require 'genevalidator/sequences'
+require 'genevalidator/query'
 
 module GeneValidator
   # Gets the raw sequences for each hit in a BLAST output file
@@ -14,34 +15,32 @@ module GeneValidator
       extend Forwardable
       def_delegators GeneValidator, :opt, :config
 
-      ##
-      # Obtains raw_sequences from BLAST output file...
-      def run
-        @opt = opt
-        @config = config
-
+      def init
         $stderr.puts 'Extracting sequences within the BLAST output file from' \
                      ' the BLAST database'
 
-        if @opt[:blast_xml_file]
-          @blast_file  = @opt[:blast_xml_file]
-        else
-          @blast_file = @opt[:blast_tabular_file]
-        end
+        @blast_file = opt[:blast_xml_file] if opt[:blast_xml_file]
+        @blast_file = opt[:blast_tabular_file] if opt[:blast_tabular_file]
 
-        @opt[:raw_sequences] = @blast_file + '.raw_seq'
-        index_file           = @blast_file + '.index'
+        opt[:raw_sequences] = @blast_file + '.raw_seq'
+        @index_file         = @blast_file + '.index'
+      end
 
+      ##
+      # Obtains raw_sequences from BLAST output file...
+      def run
+        init
         if opt[:db] =~ /remote/
-          write_a_raw_seq_file(@opt[:raw_sequences], 'remote')
+          write_a_raw_seq_file(opt[:raw_sequences], 'remote')
         else
-          write_an_index_file(index_file, 'local')
-          obtain_raw_seqs_from_local_db(index_file, @opt[:raw_sequences])
+          write_an_index_file(@index_file, 'local')
+          FetchRawSequences.extract_from_local_db(true, nil, @index_file)
         end
-        index_raw_seq_file(@opt[:raw_sequences])
+        index_raw_seq_file(opt[:raw_sequences])
       end
 
       ##
+      #
       # Index the raw sequences file...
       def index_raw_seq_file(raw_seq_file = opt[:raw_sequences])
         # leave only the identifiers in the fasta description
@@ -74,8 +73,15 @@ module GeneValidator
 
       def write_an_index_file(output_file, db_type)
         file = File.open(output_file, 'w+')
-        iterate_xml(file, db_type) if @opt[:blast_xml_file]
-        iterate_tabular(file, db_type) if @opt[:blast_tabular_file]
+        iterate_xml(file, db_type) if opt[:blast_xml_file]
+        iterate_tabular(file, db_type) if opt[:blast_tabular_file]
+      rescue
+        $stderr.puts '*** Error: There was an error in analysing the BLAST'
+        $stderr.puts '    output file. Please ensure that BLAST output file'
+        $stderr.puts '    is in the correct format and then try again. If you'
+        $stderr.puts '    are using a remote database, please ensure that you'
+        $stderr.puts '    have internet access.'
+        exit 1
       ensure
         file.close unless file.nil?
       end
@@ -83,85 +89,125 @@ module GeneValidator
       alias_method :write_a_raw_seq_file, :write_an_index_file
 
       def iterate_xml(file, db_type)
-        n = Bio::BlastXMLParser::XmlIterator.new(@opt[:blast_xml_file]).to_enum
+        n = Bio::BlastXMLParser::XmlIterator.new(opt[:blast_xml_file]).to_enum
         n.each do |iter|
           iter.each do |hit|
             if db_type == 'remote' || hit.hit_id.nil?
-              file.puts obtain_raw_seqs_from_remote_db(hit.accession)
+              file.puts FetchRawSequences.extract_from_remote_db(hit.accession)
             else
-              file.puts hit.hit_id
+              file.puts hit.accession
             end
           end
         end
-      rescue
-        $stderr.puts '*** Error: There was an error in analysing the BLAST XML file.'
-        $stderr.puts '    Please ensure that BLAST XML file is in the correct format'
-        $stderr.puts '    and then try again. If you are using a remote database,'
-        $stderr.puts '    please ensure that you have internet access.'
-        exit 1
       end
 
       def iterate_tabular(file, db_type)
-        table_headers = @opt[:blast_tabular_options].split(/[ ,]/)
-        tab_file      = File.read(@opt[:blast_tabular_file])
+        table_headers = opt[:blast_tabular_options].split(/[ ,]/)
+        tab_file      = File.read(opt[:blast_tabular_file])
         rows = CSV.parse(tab_file, col_sep: "\t",
                                    skip_lines: /^#/,
                                    headers: table_headers)
-        assert_table_has_correct_no_of_collumns(rows, table_headers)
 
         rows.each do |row|
           if db_type == 'remote' || row['sseqid'].nil?
-            file.puts obtain_raw_seqs_from_remote_db(row['sacc'])
+            file.puts FetchRawSequences.extract_from_remote_db(row['sacc'])
           else
             file.puts row['sseqid']
           end
         end
+      end
+    end
+  end
+
+  class FetchRawSequences
+    class << self
+      extend Forwardable
+      def_delegators GeneValidator, :opt, :config
+
+      def run(identifier, accession)
+        # first try to extract from previously created raw_sequences HASH
+        raw_seq = extract_from_index(identifier) if opt[:raw_sequences]
+        # then try to just extract that sequence based on accession.
+        if opt[:db] !~ /remote/ && (raw_seq.nil? || raw_seq =~ /Error/)
+          raw_seq = extract_from_local_db(false, accession)
+        end
+        # then try to extract from remote database
+        if opt[:db] =~ /remote/ && (raw_seq.nil? || raw_seq =~ /Error/)
+          raw_seq = extract_from_remote_db(accession)
+        end
+        # return nil if the raw_sequence still produces an error.
+        (raw_seq =~ /Error/) ? nil : raw_seq
+      end
+
+      ##
+      # Gets raw sequence by fasta identifier from a fasta index file
+      # Params:
+      # +identifier+: String
+      # Output:
+      # String with the nucleotide sequence corresponding to the identifier
+      def extract_from_index(identifier)
+        idx         = config[:raw_seq_file_load][identifier]
+        query       = IO.binread(opt[:raw_sequences], idx[1] - idx[0], idx[0])
+        parse_query = query.scan(/>([^\n]*)\n([A-Za-z\n]*)/)[0]
+        parse_query[1].gsub("\n", '')
       rescue
-        $stderr.puts '*** Error: There was an error in analysing the BLAST tabular'
-        $stderr.puts '    file. Please ensure that BLAST tabular file is in the correct'
-        $stderr.puts '    format and then try again. If you are using a remote'
-        $stderr.puts '    database, please ensure that you have internet access.'
-        exit 1
+        'Error' # return error so it can then try alternative fetching method.
       end
 
-      def obtain_raw_seqs_from_local_db(index_file, raw_seq_file)
-        cmd = "blastdbcmd -entry_batch '#{index_file}' -db '#{@opt[:db]}'" \
-              " -outfmt '%f' -out '#{raw_seq_file}'"
-        `#{cmd}`
+      ##
+      # Gets raw sequence by accession number from a givem database
+      # Params:
+      # +accno+: accession number as String
+      # +db+: database as String
+      # Output:
+      # String with the nucleotide sequence corresponding to the accession
+      def extract_from_local_db(batch, accno = nil, idx_file = nil)
+        cmd = (batch) ? batch_raw_seq_cmd(idx_file) : single_raw_seq_cmd(accno)
+        efile = Tempfile.new('blast_out')
+        `#{cmd} &>#{efile.path}`
+        raw_seqs = efile.read
+        failed_raw_sequences(raw_seqs) if batch && raw_seqs =~ /Error/
+        raw_seqs # when obtaining a single raw_seq, this contains the sequence
+      ensure
+        efile.close
+        efile.unlink
       end
 
-      def obtain_raw_seqs_from_remote_db(accession)
-        uri      = 'http://www.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?' \
-                   "db=protein&retmax=1&usehistory=y&term=#{accession}/"
-        result   = Net::HTTP.get(URI.parse(uri))
-        query    = result.scan(%r{<\bQueryKey\b>([\w\W\d]+)</\bQueryKey\b>})[0][0]
-        web_env  = result.scan(%r{<\bWebEnv\b>([\w\W\d]+)</\bWebEnv\b>})[0][0]
-
-        uri      = 'http://www.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?' \
-                   'rettype=fasta&retmode=text&retstart=0&retmax=1&' \
-                   "db=protein&query_key=#{query}&WebEnv=#{web_env}"
-        result   = Net::HTTP.get(URI.parse(uri))
-        raw_seqs = result[0..result.length - 2]
-        unless raw_seqs.downcase.index(/error/).nil?
-          $stderr.puts '*** Error: There was an error in obtaining the raw sequence' \
-                       ' of a BLAST hit. Please ensure that you have internet access.'
-          exit 1
-        end
-        raw_seqs
+      def batch_raw_seq_cmd(index_file)
+        "blastdbcmd -entry_batch '#{index_file}' -db '#{opt[:db]}'" \
+        " -outfmt '%f' -out '#{opt[:raw_sequences]}'"
       end
 
-      def assert_table_has_correct_no_of_collumns(rows, table_headers)
-        rows.each do |row|
-          unless row.length == table_headers.length
-            $stderr.puts '*** Error: The BLAST tabular file cannot be parsed. This is' \
-                         ' could possibly be due to an incorrect BLAST tabular' \
-                         ' options ("-o", "--blast_tabular_options") being supplied.' \
-                         ' Please correct this and try again.'
-            exit 1
+      def single_raw_seq_cmd(accession)
+        "blastdbcmd -entry '#{accession}' -db '#{opt[:db]}' -outfmt '%s'"
+      end
+
+      def failed_raw_sequences(blast_output)
+        blast_output.each_line do |line|
+          acc = line.match(/Error: (\w+): OID not found/)[1]
+          $stderr.puts "\nCould not find sequence '#{acc.chomp}' within the" \
+                       ' BLAST database.'
+          $stderr.puts "Attempting to obtain sequence '#{acc.chomp}' from" \
+                       ' remote BLAST databases.'
+          File.open(opt[:raw_sequences], 'a+') do |f|
+            f.puts extract_from_remote_db(acc)
           end
-          break # break after checking the first column
         end
       end
+
+      def extract_from_remote_db(accession, db_seq_type = 'protein')
+        uri     = 'http://www.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?' \
+                  "db=#{db_seq_type}&retmax=1&usehistory=y&term=#{accession}/"
+        result  = Net::HTTP.get(URI.parse(uri))
+        query   = result.match(%r{<\bQueryKey\b>([\w\W\d]+)</\bQueryKey\b>})[1]
+        web_env = result.match(%r{<\bWebEnv\b>([\w\W\d]+)</\bWebEnv\b>})[1]
+
+        uri     = 'http://www.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?' \
+                  'rettype=fasta&retmode=text&retstart=0&retmax=1&' \
+                  "db=#{db_seq_type}&query_key=#{query}&WebEnv=#{web_env}"
+        result  = Net::HTTP.get(URI.parse(uri))
+        result[0..result.length - 2]
+      end
     end
   end
 end

data/lib/genevalidator/hsp.rb

@@ -1,5 +1,6 @@
 require 'genevalidator/blast'
 require 'genevalidator/exceptions'
+
 module GeneValidator
   # A class that initialises the BLAST tabular attributes
   class Hsp
@@ -45,14 +46,13 @@ module GeneValidator
       @pidentity           = hash['pident'].to_f if hash['pident']
       @identity            = hash['nident'].to_f if hash['nident']
       @hsp_evalue          = hash['evalue'].to_f if hash['evalue']
-      if hash['qseq']
-        query_seq_type = BlastUtils.guess_sequence_type(@query_alignment)
-        fail SequenceTypeError if query_seq_type != :protein
-      end
-      if hash['sseq']
-        hit_seq_type = BlastUtils.guess_sequence_type(@hit_alignment)
-        fail SequenceTypeError if hit_seq_type != :protein
-      end
+      assert_seq_type(@query_alignment) if hash['sseq']
+      assert_seq_type(@hit_alignment) if hash['sseq']
+    end
+
+    def assert_seq_type(query)
+      seq_type = BlastUtils.guess_sequence_type(query)
+      fail SequenceTypeError if seq_type != :protein
     end
   end
 end

data/lib/genevalidator/json_to_gv_results.rb

@@ -93,16 +93,16 @@ module GeneValidator
         end
       end
 
-      # Since the whole idea is that users would sort by
+      # Since the whole idea is that users would sort by JSON when there are
+      # too many rows in the HTML table.
       def turn_off_automated_sorting
         script_file = File.join(@config[:html_path], 'files/js/script.js')
-        temp_file   = File.join(@config[:html_path], 'files/js/script.temp.js')
-        File.open(temp_file, 'w') do |out_file|
+        File.open("#{script_file}.tmp", 'w') do |out_file|
           out_file.puts File.readlines(script_file)[0..23].join
           out_file.puts '}'
           out_file.puts File.readlines(script_file)[26..-1].join
         end
-        FileUtils.mv(temp_file, script_file)
+        FileUtils.mv("#{script_file}.tmp", script_file)
       end
     end
   end

data/lib/genevalidator/output.rb

@@ -34,8 +34,6 @@ module GeneValidator
       @prediction_def = definition
       @nr_hits        = no_of_hits
       @idx            = current_idx
-
-      @app_html       = File.join(@config[:html_path], 'files/table.html')
     end
 
     def print_output_console
@@ -50,33 +48,31 @@ module GeneValidator
 
     def print_console_header
       @config[:console_header_printed] = true
-      print format("%3s\t%5s\t%20s\t%7s\t", 'No', 'Score', 'Identifier', 'No_Hits')
+      print format("%3s\t%5s\t%20s\t%7s\t", 'No', 'Score', 'Identifier',
+                   'No_Hits')
       puts validations.map(&:short_header).join("\t")
     end
 
     def generate_html
       mutex_html.synchronize do
-        output_html = output_filename
+        output_html   = output_filename
         query_erb     = File.join(@config[:aux], 'template_query.erb')
         template_file = File.open(query_erb, 'r').read
         erb           = ERB.new(template_file, 0, '>')
         File.open(output_html, 'a') { |f| f.write(erb.result(binding)) }
-        File.open(@app_html, 'a') { |f| f.write(erb.result(binding)) }
       end
     end
 
     def output_filename
-      i = (@config[:run_no].to_f / @config[:output_max]).ceil
-      output_html = File.join(@config[:html_path], "results#{i}.html")
+      idx         = (@config[:run_no].to_f / @config[:output_max]).ceil
+      output_html = File.join(@config[:html_path], "results#{idx}.html")
       write_html_header(output_html)
       output_html
     end
 
     def write_html_header(output_html)
       head_erb       = File.join(@config[:aux], 'template_header.erb')
-      head_table_erb = File.join(@config[:aux], 'app_template_header.erb')
       set_up_html(head_erb, output_html) unless File.exist?(output_html)
-      set_up_html(head_table_erb, @app_html) unless File.exist?(@app_html)
     end
 
     def set_up_html(erb_file, output_file)
@@ -99,25 +95,31 @@ module GeneValidator
     def create_validation_hashes(row)
       row[:validations] = {}
       @validations.each do |item|
-        val = { header: item.header, description: item.description,
-                status: item.color, print: item.print.gsub(' ', ' ') }
-        if item.color != 'warning'
-          explain = { approach: item.approach, explanation: item.explanation,
-                      conclusion: item.conclusion }
-          val.merge!(explain)
-        end
+        val     = add_basic_validation_info(item)
+        explain = add_explanation_data(item) if item.color != 'warning'
+        val.merge!(explain) if explain
         val[:graphs] = create_graphs_hash(item) unless item.plot_files.nil?
         row[:validations][item.short_header] = val
       end
       row
     end
 
+    def add_basic_validation_info(item)
+      { header: item.header, description: item.description, status: item.color,
+        print: item.print.gsub(' ', ' ') }
+    end
+
+    def add_explanation_data(item)
+      { approach: item.approach, explanation: item.explanation,
+        conclusion: item.conclusion }
+    end
+
     def create_graphs_hash(item)
       graphs = []
       item.plot_files.each do |g|
         graphs << { data: g.data, type: g.type, title: g.title,
-                    footer: g.footer, xtitle: g.xtitle,
-                    ytitle: g.ytitle, aux1: g.aux1, aux2: g.aux2 }
+                    footer: g.footer, xtitle: g.xtitle, ytitle: g.ytitle,
+                    aux1: g.aux1, aux2: g.aux2 }
       end
       graphs
     end
@@ -140,16 +142,9 @@ module GeneValidator
     # +html_path+: path of the html folder
     # +filemane+: name of the fasta input file
     def self.print_footer(overview, config)
-      overall_evaluation = overview(overview)
-
-      create_plot_json(overview[:scores], config[:plot_dir])
+      set_overall_evaluation(overview, config)
 
-      less = overall_evaluation[0].gsub("\n", '<br>').gsub("'", %q(\\\'))
-
-      eval = print_summary_to_console(overall_evaluation, config[:summary])
-      evaluation     = eval.gsub("\n", '<br>').gsub("'", %q(\\\'))
-
-      footer_erb     = File.join(config[:aux], 'template_footer.erb')
+      footer_erb          = File.join(config[:aux], 'template_footer.erb')
 
       no_of_results_files = (config[:run_no].to_f / config[:output_max]).ceil
       template_file       = File.open(footer_erb, 'r').read
@@ -164,40 +159,44 @@ module GeneValidator
       end
 
       turn_off_sorting(config[:html_path]) if no_of_results_files > 1
+    end
+
+    def self.set_overall_evaluation(overview, config)
+      overall_evaluation = overview(overview)
+      less = overall_evaluation[0].gsub("\n", '<br>').gsub("'", %q(\\\'))
+
+      eval = print_summary_to_console(overall_evaluation, config[:summary])
+      evaluation     = eval.gsub("\n", '<br>').gsub("'", %q(\\\'))
 
-      # write footer for the app
-      app_footer_erb = File.join(config[:aux], 'app_template_footer.erb')
-      table_html     = File.join(config[:html_path], 'files/table.html')
-      table_footer_template = File.open(app_footer_erb, 'r').read
-      table_erb             = ERB.new(table_footer_template, 0, '>')
-      File.open(table_html, 'a+') { |f| f.write(table_erb.result(binding)) }
+      create_overview_json(overview[:scores], config[:plot_dir], less,
+                           evaluation)
     end
 
     def self.turn_off_sorting(html_path)
       script_file = File.join(html_path, 'files/js/script.js')
-      temp_file   = File.join(html_path, 'files/js/script.temp.js')
-      File.open(temp_file, 'w') do |out_file|
-        out_file.puts File.readlines(script_file)[30..-1].join
+      File.open("#{script_file}.tmp", 'w') do |f|
+        f.puts File.readlines(script_file)[30..-1].join
       end
-      FileUtils.mv(temp_file, script_file)
+      FileUtils.mv("#{script_file}.tmp", script_file)
     end
 
     def self.print_summary_to_console(overall_evaluation, summary)
       # print to console
       eval = ''
-      overall_evaluation.each { |e| eval << "\n#{e}" }
+      overall_evaluation.each { |e| eval << "#{e}\n" }
       $stderr.puts eval if summary
       $stderr.puts ''
       eval
     end
 
     # make the historgram with the resulted scores
-    def self.create_plot_json(scores, plot_dir)
+    def self.create_overview_json(scores, plot_dir, less, evaluation)
       plot_file = File.join(plot_dir, 'overview.json')
       data = [scores.group_by { |a| a }.map { |k, vs| { 'key' => k, 'value' => vs.length, 'main' => false } }]
       hash = { data: data, type: :simplebars, title: 'Overall Evaluation',
                footer: '', xtitle: 'Validation Score',
-               ytitle: 'Number of Queries', aux1: 10, aux2: '' }
+               ytitle: 'Number of Queries', aux1: 10, aux2: '', less: less,
+               evaluation: evaluation }
       File.open(plot_file, 'w') { |f| f.write hash.to_json }
     end
 
@@ -233,8 +232,8 @@ module GeneValidator
       eval
     end
 
+    # errors per validation
     def self.errors_overview(o)
-      # errors per validation
       error_eval = ''
       o[:map_errors].each do |k, v|
         error_eval << "\nWe couldn't run #{k} Validation for #{v} queries"