genevalidator 1.6.12 → 2.1.3

data/exemplar_data/README.md

@@ -0,0 +1,60 @@
+# Running GeneValidator with sample data
+
+Here, we walk through the steps involved in analysing some sample data with GeneValidator. There are two options on how to run genevalidator - the second option is faster with larger input files.
+
+## Expected Results
+
+<strong>protein_data.fa</strong> [See here](http://wurmlab.github.io/tools/genevalidator/examplar_data/protein_input/)
+<strong>mrna_data.fa</strong> [See here](http://wurmlab.github.io/tools/genevalidator/examplar_data/genetic_input/)
+
+##### Running GeneValidator with a the included SwissProt Database, with four threads
+
+```bash
+# Protein data
+$ genevalidator -n 4 protein_data.fa
+
+# MRNA data
+$ genevalidator -n 4 mrna_data.fa
+```
+
+This will produce a folder that will contain your result files.
+
+##### Running GeneValidator with a pre-computed BLAST XML file
+
+For protein_data.fa:
+
+```
+blastp -db DATABASE_PATH -num_threads 4 -out protein_data.blast.xml -query protein_data.fa -outfmt 5
+
+# Run GeneValidator
+genevalidator -d DATABASE_PATH -n 4 -x protein_data.blast.xml protein_data.fa
+```
+
+For mrna_data.fa:
+
+```
+blastx -db DATABASE_PATH -num_threads 4 -out mrna_data.blast.xml -query mrna_data.fa -outfmt 5
+
+# Run GeneValidator
+genevalidator -d DATABASE_PATH -n 4 -x mrna_data.blast.xml mrna_data.fa
+```
+
+##### Running GeneValidator with a pre-computed BLAST tabular file
+
+For protein_data.fa:
+
+```
+blastp -db DATABASE_PATH -num_threads 4 -out protein_data.blast.tsv -query protein_data.fa -outfmt '7 qseqid sseqid sacc slen qstart qend sstart send length qframe pident nident evalue qseq sseq'
+
+# Run GeneValidator
+genevalidator -d DATABASE_PATH -n 4 -t protein_data.blast.tsv --blast_tabular_options 'qseqid sseqid sacc slen qstart qend sstart send length qframe pident nident evalue qseq sseq' protein_data.fa
+```
+
+For mrna_data.fa:
+
+```
+blastp -db DATABASE_PATH -num_threads 4 -out mrna_data.blast.tsv -query mrna_data.fa -outfmt '7 qseqid sseqid sacc slen qstart qend sstart send length qframe pident nident evalue qseq sseq'
+
+# Run GeneValidator
+genevalidator -d DATABASE_PATH -n 4 -t mrna_data.blast.tsv --blast_tabular_options 'qseqid sseqid sacc slen qstart qend sstart send length qframe pident nident evalue qseq sseq' mrna_data.fa
+```

data/{data/mrna_data.fasta → exemplar_data/mrna_data.fa}

@@ -235,4 +235,4 @@ CCATGCCGGAGCATCAGTAGATCTTGCCATCTTCTCCCTTCATCTGGCAGGTGTCTCCTC
 CATCCTCGGAGCAATTAACTTTATCACCACAGCCATCAACATGAAACCACCTGCCCTCTC
 ACAATACCAAACCCCCCTATTCGTTTGATCCGTCTTAATTACCGCCATCCTTCTTCTCCT
 TTCCCTCCCAGTTCTCGCCGCTGGTATTACAATGCTTCTAACAGATCGAAATCTAAACAC
-TACATTCTTCGACCCTGCAGGGGGCGGAGACCCAATTTTATACCAACACTTA
+TACATTCTTCGACCCTGCAGGGGGCGGAGACCCAATTTTATACCAACACTTA

data/genevalidator.gemspec

@@ -1,12 +1,10 @@
-# coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'genevalidator/version'
 
 Gem::Specification.new do |s|
-  # meta
   s.name        = 'genevalidator'
-  s.version     =  GeneValidator::VERSION
+  s.version     = GeneValidator::VERSION
   s.authors     = ['Monica Dragan', 'Ismail Moghul', 'Anurag Priyam',
                    'Yannick Wurm']
   s.email       = 'y.wurm@qmul.ac.uk'
@@ -14,19 +12,27 @@ Gem::Specification.new do |s|
   s.license     = 'AGPL'
   s.summary     = 'Identifying problems with gene predictions.'
   s.description = 'The tool validates the input predicted genes and provides' \
-                  ' useful information (length validation, gene merge'\
+                  ' useful information (length validation, gene merge' \
                   ' validation, sequence duplication checking, ORF finding)' \
                   ' based on the similarities to genes in public databases.'
+  s.required_ruby_version = '>= 2.2.0'
 
-  s.required_ruby_version = '>= 2.0.0'
-  s.add_development_dependency 'bundler', '~> 1.6'
-  s.add_development_dependency 'rake', '~>10.3'
-  s.add_development_dependency 'yard', '~> 0.8'
-  s.add_development_dependency 'codeclimate-test-reporter', '~> 0.4', '>= 0.4.7'
-  s.add_development_dependency('minitest', '~> 5.4')
-  s.add_dependency('bio', '~> 1.4')
-  s.add_dependency('bio-blastxmlparser', '~>2.0')
-  s.add_dependency('statsample', '2.0.1')
+  s.add_development_dependency 'minitest', '~> 5.10'
+  s.add_development_dependency 'rake', '~> 12.3'
+  s.add_development_dependency 'yard', '~> 0.9.11'
+
+  s.add_dependency 'bio', '~> 1.4'
+  s.add_dependency 'bio-blastxmlparser', '~> 2.0'
+  s.add_dependency 'genevalidatorapp', '~> 2.1.3'
+  s.add_dependency 'rack', '~> 2.0'
+  s.add_dependency 'slim', '~>3.0'
+  s.add_dependency 'statsample', '2.1.0'
+
+  # Adding mechanize gem just to silence a message on load.
+  # This is due the Statsample gem
+  # See https://github.com/SciRuby/daru/issues/404
+  # See https://github.com/SciRuby/statsample/pull/69
+  s.add_dependency 'mechanize', '2.7.5'
 
   s.files         = `git ls-files -z`.split("\x0")
   s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
@@ -35,15 +41,24 @@ Gem::Specification.new do |s|
 
   s.post_install_message = <<INFO
 
-------------------------------------------------------------------------
-  Thank you for validating your gene predictions with GeneValidator!
+----------------------------------------------------------------------------
+Thank you for validating your gene predictions with GeneValidator!
+
+==> To launch GeneValidator execute 'genevalidator' from command line.
+
+        genevalidator [OPTIONAL ARGUMENTS] INPUT_FILE
+
+    See 'genevalidator --help' for more information
+
+==> To launch GeneValidator as a web application execute 'genevalidator' from command line.
+
+        genevalidator app [OPTIONAL ARGUMENTS]
 
-  To launch GeneValidator execute 'genevalidator' from command line.
+    See 'genevalidator app --help' for more information
 
-    $ genevalidator [options] FASTA_FILE
+==> Visit https://wurmlab.github.io/tools/genevalidator/ for more information.
 
-  Visit https://github.com/wurmlab/GeneValidator for more information.
-------------------------------------------------------------------------
+----------------------------------------------------------------------------
 
 INFO
 end

data/install.sh

@@ -0,0 +1,92 @@
+#!/bin/sh
+
+## USAGE: bash install.sh $INSTALL_DIR
+##     $ bash install.sh $INSTALL_DIR
+
+set -eu
+
+# OS detection
+KERNEL="$(uname -s | tr '[:upper:]' '[:lower:]')"
+
+if [ "$KERNEL" = "darwin" ]; then
+  PLATFORM='osx'
+elif [ "$KERNEL" = "linux" ]; then
+  ARCH=$(uname -m)
+  if [ "$ARCH" = "x86_64" ]; then
+    PLATFORM='linux-x86_64'
+  else
+    PLATFORM='linux-x86'
+  fi
+fi
+
+# If there is an argument then there is where GV will installed
+if [ "$0" = 'sh' ]; then
+  # I.e. when piping from curl
+  INSTALL_DIR=$PWD/genevalidator
+elif [ "$0" = 'install.sh' ]; then
+  # I.e. when running directly
+  INSTALL_DIR=$PWD/genevalidator
+else
+  INSTALL_DIR="$0"
+fi
+
+GV_URL=$(curl -s https://api.github.com/repos/wurmlab/genevalidator/releases/latest \
+  | grep browser_download_url \
+  | grep -i $PLATFORM \
+  | cut -d '"' -f 4)
+
+echo >&2 "==> Installing GeneValidator to:"
+echo >&2 "    ${INSTALL_DIR}"
+echo >&2
+
+mkdir "${INSTALL_DIR}"
+curl -SL "$GV_URL" | tar zxf - -C "${INSTALL_DIR}" --strip-components 1
+
+echo >&2
+echo >&2 "==> GeneValidator successfully installed."
+
+### Check which SHELL and then test different profile files
+case $SHELL in
+*/zsh)
+  # assume Zsh
+  if test -e "${HOME}/.zshrc"; then
+    DOT_FILE=${HOME}/.zshrc
+  elif test -e "${HOME}/.zprofile"; then
+    DOT_FILE=${HOME}/.zprofile
+  elif test -e "${HOME}/.profile"; then
+    DOT_FILE=${HOME}/.profile
+  fi
+  ;;
+*/bash)
+  # assume Bash
+  if test -e "${HOME}/.bashrc"; then
+    DOT_FILE=${HOME}/.bashrc
+  elif test -e "${HOME}/.bash_profile"; then
+    DOT_FILE=${HOME}/.bash_profile
+  elif test -e "${HOME}/.profile"; then
+    DOT_FILE=${HOME}/.profile
+  fi
+  ;;
+*)
+  if test -e "${HOME}/.profile"; then
+    DOT_FILE=${HOME}/.profile
+  fi
+esac
+
+
+if [ -z ${DOT_FILE+x} ]; then
+  # DOT File hasn't been set.
+  echo >&2
+  echo >&2 '==> No profile files were found.'
+  echo >&2 '    Please create one and add the following line to that file:'
+  echo >&2
+  echo >&2 '    export PATH="'"${INSTALL_DIR}"'/bin:${PATH}"'
+else
+  echo >&2 'export PATH="'"${INSTALL_DIR}"'/bin:${PATH}"' >> "${DOT_FILE}"
+  echo >&2
+  echo >&2 "==> Added GeneValidator to your PATH in ${DOT_FILE}"
+  echo >&2
+  echo >&2 "==> Run \`genevalidator -h\` in a new window to get started."
+fi
+
+echo >&2

data/lib/genevalidator.rb

@@ -5,61 +5,36 @@ require 'genevalidator/arg_validation'
 require 'genevalidator/blast'
 require 'genevalidator/exceptions'
 require 'genevalidator/get_raw_sequences'
+require 'genevalidator/json_to_gv_results'
 require 'genevalidator/output'
+require 'genevalidator/output_files'
 require 'genevalidator/tabular_parser'
 require 'genevalidator/validation'
 
 # Top level module / namespace.
 module GeneValidator
   class << self
-    attr_accessor :opt, :config, :overview
+    attr_accessor :opt, :config, :overview, :dirs
     attr_reader :raw_seq_file_index
     attr_reader :raw_seq_file_load
     # array of indexes for the start offsets of each query in the fasta file
     attr_reader :query_idx
-    attr_accessor :mutex, :mutex_html, :mutex_json, :mutex_array
+    attr_accessor :mutex, :mutex_array
 
-    def init(opt, start_idx = 1, summary = true)
-      $stderr.puts 'Analysing input arguments'
+    def init(opt, start_idx = 1)
+      warn '==> Analysing input arguments'
       @opt = opt
       GVArgValidation.validate_args # validates @opt
+      number_of_sequences = index_the_input
 
-      @config = {
-        idx: 0,
-        start_idx: start_idx,
-        summary: summary,
-
-        type: BlastUtils.guess_sequence_type_from_input_file,
-        filename: File.basename(@opt[:input_fasta_file]),
-        html_path: "#{@opt[:input_fasta_file]}.html",
-        json_file: File.join(File.dirname(@opt[:input_fasta_file]),
-                             "#{File.basename(@opt[:input_fasta_file])}.json"),
-        plot_dir: "#{@opt[:input_fasta_file]}.html/files/json",
-        aux: File.expand_path(File.join(File.dirname(__FILE__), '../aux')),
-
-        json_output: [],
-        run_no: 0,
-        output_max: 2500 # max no. of queries in the output file
-      }
-
-      @overview = {
-        no_queries: 0,
-        scores: [],
-        good_scores: 0,
-        bad_scores: 0,
-        nee: 0,
-        no_mafft: 0,
-        no_internet: 0,
-        map_errors: Hash.new(0),
-        run_time: Hash.new(Pair1.new(0, 0))
-      }
+      @config = setup_config(start_idx, number_of_sequences)
+      @dirs = setup_dirnames(@opt[:input_fasta_file])
 
       @mutex       = Mutex.new
       @mutex_array = Mutex.new
-      @mutex_html  = Mutex.new
-      @mutex_json  = Mutex.new
-      create_output_folder
-      index_the_input
+
+      resume_from_previous_run(opt[:resumable]) unless opt[:resumable].nil?
+
       RawSequences.index_raw_seq_file if @opt[:raw_sequences]
     end
 
@@ -69,6 +44,8 @@ module GeneValidator
       # Run BLAST on all sequences (generates @opt[:blast_xml_file])
       # if no BLAST OUTPUT file provided...
       unless @opt[:blast_xml_file] || @opt[:blast_tabular_file]
+        blast_xml_fname = "#{dirs[:filename]}.blast_xml"
+        opt[:blast_xml_file] = File.join(dirs[:tmp_dir], blast_xml_fname)
         BlastUtils.run_blast_on_input_file
       end
       # Obtain fasta file of all BLAST hits if running align or dup validations
@@ -78,19 +55,110 @@ module GeneValidator
       end
       # Run Validations
       iterator = parse_blast_output_file
-      (Validations.new).run_validations(iterator)
+      Validations.new.run_validations(iterator)
+      produce_output
+      print_directories_locations
+    end
+
+    ##
+    # Params:
+    # +output+: filename or stream, according to the type
+    # +type+: file or stream
+    # Returns an iterator..
+    def parse_blast_output_file
+      if @opt[:blast_xml_file]
+        Bio::BlastXMLParser::XmlIterator.new(@opt[:blast_xml_file]).to_enum
+      else
+        TabularParser.new
+      end
+      ## TODO: Add a Rescue statement - e.g. if unable to create the Object...
+    end
+
+    # Also called by json_to_gv script
+    def setup_dirnames(input_file)
+      fname = File.basename(input_file, File.extname(input_file))
+      out_dir = setup_output_dir(fname)
+      { filename: fname,
+        output_dir: out_dir,
+        tmp_dir: File.join(out_dir, 'tmp'),
+        json_dir:  File.join(out_dir, 'tmp/json'),
+        html_file: File.join(out_dir, "#{fname}_results*.html"),
+        json_file: File.join(out_dir, "#{fname}_results.json"),
+        csv_file: File.join(out_dir, "#{fname}_results.csv"),
+        summary_file: File.join(out_dir, "#{fname}_summary.csv"),
+        fasta_file: File.join(out_dir, "#{fname}_results.fa"),
+        aux_dir: File.expand_path('../aux', __dir__) }
+    end
+
+    def extract_input_fasta_sequence(index)
+      start_offset = @query_idx[index + 1] - @query_idx[index]
+      end_offset = @query_idx[index]
+      IO.binread(@opt[:input_fasta_file], start_offset, end_offset)
+    end
+
+    def produce_output
+      @overview = Output.generate_overview(@config[:json_output],
+                                           @opt[:min_blast_hits])
+      eval_text = Output.generate_evaluation_text(@overview)
+      Output.print_console_footer(eval_text, @opt)
+
+      output_files = OutputFiles.new
+      output_files.write_json
+      output_files.write_html(eval_text)
+      output_files.write_csv
+      output_files.write_summary
+      output_files.print_best_fasta
+    end
+
+    private
+
+    def setup_config(start_idx, seq_length)
+      {
+        idx: 0,
+        start_idx: start_idx,
+
+        type: BlastUtils.guess_sequence_type_from_input_file,
 
-      Output.write_json_file(@config[:json_output], @config[:json_file])
-      Output.print_footer(@overview, @config)
+        json_output: Array.new(seq_length),
+        run_no: 0,
+        output_max: 2500 # max no. of queries in the output html file
+      }
     end
 
     ##
     # Creates the output folder and copies the auxiliar folders to this folder
-    def create_output_folder(output_dir = @config[:html_path],
-                             aux_dir = @config[:aux])
+    def setup_output_dir(fname)
+      dir_name = "#{fname}_" + Time.now.strftime('%Y_%m_%d_%H_%M_%S')
+      default_outdir = File.join(Dir.pwd, dir_name)
+      output_dir = @opt[:output_dir].nil? ? default_outdir : @opt[:output_dir]
+      assert_output_dir_does_not_exist(output_dir)
       Dir.mkdir(output_dir)
-      aux_files = File.join(aux_dir, 'files/')
-      FileUtils.cp_r(aux_files, output_dir)
+      Dir.mkdir(File.join(output_dir, 'tmp'))
+      cp_html_files(output_dir)
+      output_dir
+    end
+
+    def assert_output_dir_does_not_exist(output_dir)
+      return unless Dir.exist?(output_dir)
+      FileUtils.rm_r(output_dir) if @opt[:force_rewrite]
+      return if @opt[:force_rewrite]
+      warn "The output directory (#{output_dir}) already exists."
+      warn ''
+      warn 'Please remove this directory before continuing.'
+      warn 'Alternatively, you rerun GeneValidator with the `--force` argument,'
+      warn 'which rewrites over any previous output.'
+      exit 1
+    end
+
+    def cp_html_files(output_dir)
+      if @opt[:output_formats].include? 'html'
+        aux_files = File.expand_path('../aux/html_files/', __dir__)
+        FileUtils.cp_r(aux_files, output_dir)
+        FileUtils.ln_s(File.join('..', 'html_files', 'json'),
+                       File.join(output_dir, 'tmp', 'json'))
+      else
+        Dir.mkdir(File.join(output_dir, 'tmp', 'json'))
+      end
     end
 
     ##
@@ -99,22 +167,69 @@ module GeneValidator
     # start and end positions of each query.
     def index_the_input
       fasta_content = IO.binread(@opt[:input_fasta_file])
-      @query_idx = fasta_content.enum_for(:scan, /(>[^>]+)/).map { Regexp.last_match.begin(0) }
+      @query_idx = fasta_content.enum_for(:scan, /(>[^>]+)/).map do
+        Regexp.last_match.begin(0)
+      end
       @query_idx.push(fasta_content.length)
+      @query_idx.length - 1
     end
 
-    ##
-    # Params:
-    # +output+: filename or stream, according to the type
-    # +type+: file or stream
-    # Returns an iterator..
-    def parse_blast_output_file
-      if @opt[:blast_xml_file]
-        Bio::BlastXMLParser::XmlIterator.new(@opt[:blast_xml_file]).to_enum
-      else
-        TabularParser.new
+    def print_directories_locations
+      warn '==> GeneValidator output files have been saved to:'
+      warn "    #{File.expand_path(@dirs[:output_dir])}"
+    end
+
+    def resume_from_previous_run(prev_dir)
+      prev_tmp_dir = File.join(prev_dir, 'tmp')
+      return unless Dir.exist? prev_tmp_dir
+      copy_blast_xml_files(prev_tmp_dir)
+      copy_raw_seq_files(prev_tmp_dir)
+      copy_prev_json_output(prev_tmp_dir)
+    end
+
+    def copy_blast_xml_files(prev_tmp_dir)
+      return if @opt[:blast_xml_file] || @opt[:blast_tabular_file]
+      prev_blast_xml = Dir[File.join(prev_tmp_dir, '*blast_xml')]
+      return if prev_blast_xml.empty?
+      blast_xml_fname = "#{@dirs[:filename]}.blast_xml"
+      @opt[:blast_xml_file] = File.join(@dirs[:tmp_dir], blast_xml_fname)
+      FileUtils.cp(prev_blast_xml[0], @opt[:blast_xml_file])
+    end
+
+    def copy_raw_seq_files(prev_tmp_dir)
+      return if @opt[:raw_sequences]
+      return unless @opt[:validations].include?('align') ||
+                    @opt[:validations].include?('dup')
+      prev_raw_seq = Dir[File.join(prev_tmp_dir, '*raw_seq')]
+      return if prev_raw_seq.empty?
+      raw_seq_fname = "#{@dirs[:filename]}.blast_xml.raw_seq"
+      @opt[:raw_sequences] = File.join(@dirs[:tmp_dir], raw_seq_fname)
+      FileUtils.cp(prev_raw_seq[0], @opt[:raw_sequences])
+    end
+
+    def copy_prev_json_output(prev_tmp_dir)
+      prev_json_dir = File.join(prev_tmp_dir, 'json')
+      return unless Dir.exist? prev_json_dir
+      all_jsons = Dir[File.join(prev_json_dir, '*.json')]
+      FileUtils.cp(all_jsons, @dirs[:json_dir])
+      overview_json = Dir[File.join(prev_json_dir, 'overview.json')]
+      data_jsons = all_jsons - overview_json
+      parse_prev_json(data_jsons)
+    end
+
+    def parse_prev_json(data_jsons)
+      data_jsons.each do |json|
+        json_contents = File.read(File.expand_path(json))
+        data = JSON.parse(json_contents, symbolize_names: true)
+        idx = json.match(/(\d+).json/)[1].to_i - 1
+        @config[:json_output][idx] = data
+        print_prev_json_to_console(data)
       end
-      ## TODO: Add a Rescue statement - e.g. if unable to create the Object...
+    end
+
+    def print_prev_json_to_console(data)
+      JsonToGVResults.print_console_header(data)
+      JsonToGVResults.print_output_console(data)
     end
   end
 end