crb-blast 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f7c5625ff9b0713d25b0c8a4c28c56aa1316cb3
-  data.tar.gz: eaebeeedcfcc156bc7743a47683e8ce509abbe38
+  metadata.gz: 8ae488cec70a923add9cbe9e6f4a4fe0d69471d8
+  data.tar.gz: 7628e84ef78228bcc268d7deb6eb2a667131d405
 SHA512:
-  metadata.gz: 6fc0fbe6d7f2399a68409e6211f013a457a95937665d6acd391d809df0c829583e6d7f5f1b6a12cf50aa6310731f1e206c5f25349100b1e295f37f1ea5a75baf
-  data.tar.gz: 8519e10ea5de7822542d0ce05d94dbe956e9151237b49bcab89bbab74d460acfa93f9beed8ce9dcf1816b5d49365d837daf87d48c998cfde4a14266053e2f142
+  metadata.gz: f52df39062d0adf04187fca27635a4822fd2cdfb41944518773985bc08504243c02bd63564fa44656cf910909c9d3d1004c56a6f251d9918d07d2ef88f38eb2f
+  data.tar.gz: 1e46630e56d52f9056ab92a27eb33171aee066885b418855f9c9b61afccba31b5f34f2de2b53a0592d564e710cef3cf7f1f46a6eb148e57ffbb76b1025f24110

data/.gitignore

@@ -0,0 +1,13 @@
+*.lock
+*.nhr
+*.nin
+*.nsq
+*.phr
+*.pin
+*.psq
+*.blast
+coverage
+*~
+*.gem
+*fa
+*tsv

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,95 @@
+CRB-BLAST
+=========
+
+Conditional Reciprocal Best BLAST - high confidence ortholog assignment.
+
+### What is Conditional Reciprocal Best BLAST?
+
+CRB-BLAST is a novel method for finding orthologs between one set of sequences and another. This is particularly useful in genome and transcriptome annotation.
+
+CRB-BLAST initially performs a standard reciprocal best BLAST. It does this by performing BLAST alignments of query->target and target->query. Reciprocal best BLAST hits are those where the best match for any given query sequence in the query->target alignment is also the best hit of the match in the reverse (target->query) alignment.
+
+Reciprocal best BLAST is a very conservative way to assign orthologs. The main innovation in CRB-BLAST is to learn an appropriate e-value cutoff to apply to each pairwise alignment by taking into account the overall relatedness of the two datasets being compared. This is done by fitting a function to the distribution of alignment e-values over sequence lengths. The function provides the e-value cutoff for a sequence of given length.
+
+CRB-BLAST greatly improves the accuracy of ortholog assignment for de-novo transcriptome assembly ([Aubry et al. 2014](http://www.plosgenetics.org/article/info%3Adoi%2F10.1371%2Fjournal.pgen.1004365)).
+
+The CRB-BLAST algorithm was designed by [Steve Kelly](http://www.stevekellylab.com), and this implementation is by Chris Boursnell and Richard Smith-Unna. The original reference implementation from the paper is available for online use at http://www.stevekellylab.com/software/conditional-orthology-assignment.
+
+### Installation
+
+To install CRB-BLAST, simply use rubygems:
+
+`gem install crb-blast`
+
+### Prerequisites
+
+ - NCBI BLAST+ (preferably the latest version) should be installed and in your PATH.
+ - Ruby v2.0 or later. If you don't have Ruby, we suggest installing it with [RVM](http://rvm.io).
+
+`\curl -sSL https://get.rvm.io | bash -s stable --ruby`
+
+
+### Usage
+
+CRB-BLAST can be run from the command-line as a standalone program, or used as a library in your own code.
+
+#### Command-line usage
+
+CRB-BLAST can be run from the command line with:
+
+```
+crb-blast
+```
+
+The options are
+
+```
+    --query, -q <s>:   query fasta file in nucleotide format
+   --target, -t <s>:   target fasta file as nucleotide or protein
+   --evalue, -e <f>:   e-value cut off for BLAST. Format 1e-5 (default: 1.0e-05)
+  --threads, -h <i>:   number of threads to run BLAST with (default: 1)
+   --output, -o <s>:   output file as tsv
+        --split, -s:   split the fasta files into chunks and run multiple blast
+                       jobs and then combine them.
+         --help, -l:   Show this message
+```
+
+An example command is:
+
+```bash
+crb-blast --query assembly.fa --target reference_proteins.fa --threads 8 --output annotation.tsv
+```
+
+#### Library usage
+
+To include the gem in your code just `require 'crb-blast'`
+
+A quick example:
+
+```ruby
+blaster = CRB_Blast.new('test/query.fasta', 'test/target.fasta')
+blaster.run(1e-5, 4, true) # to run with an evalue cutoff of 1e-5 and 4 threads
+```
+
+A longer example with each step at a time:
+
+```ruby
+blaster = CRB_Blast.new('test/query.fasta', 'test/target.fasta')
+blaster.makedb
+blaster.run_blast(1e-5, 6, true)
+blaster.load_outputs
+blaster.find_reciprocals
+blaster.find_secondaries
+```
+
+### Getting help
+
+Please use the issue tracker if you find bugs or have trouble running CRB-BLAST.
+
+Chris Boursnell <cmb211@cam.ac.uk> maintains this software.
+
+### License
+
+This is adademic software - please cite us if you use it in your work.
+
+CRB-BLAST is released under the MIT license.

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+desc "Run tests"
+task :default => :test

data/bin/crb-blast

@@ -11,7 +11,7 @@ require 'bindeps'
 opts = Trollop::options do
   banner <<-EOS
 
-CRB-Blast v0.3 by Chris Boursnell <cmb211@cam.ac.uk>
+CRB-Blast v0.3.2 by Chris Boursnell <cmb211@cam.ac.uk>
 
 Conditional Reciprocal Best BLAST
 
@@ -48,7 +48,7 @@ EOS
 
   opt :split,
       "split the fasta files into chunks and run multiple blast jobs and then"+
-      "combine them."
+      " combine them."
 end
 
 Trollop::die :query, "must exist" if !File.exist?(opts[:query])
@@ -58,7 +58,7 @@ gem_dir = Gem.loaded_specs['crb-blast'].full_gem_path
 gem_deps = File.join(gem_dir, 'deps', 'deps.yaml')
 Bindeps.require gem_deps
 
-blaster = CRB_Blast.new(opts.query, opts.target)
+blaster = CRB_Blast::CRB_Blast.new(opts.query, opts.target)
 dbs = blaster.makedb
 run = blaster.run_blast(opts.evalue, opts.threads, opts.split)
 load = blaster.load_outputs

data/build

@@ -0,0 +1 @@
+gem build crb-blast.gemspec

data/crb-blast.gemspec

@@ -0,0 +1,26 @@
+Gem::Specification.new do |gem|
+  gem.name        = 'crb-blast'
+  gem.version     = '0.4.0'
+  gem.date        = '2014-07-23'
+  gem.summary     = "Run conditional reciprocal best blast"
+  gem.description = "See summary"
+  gem.authors     = ["Chris Boursnell", "Richard Smith-Unna"]
+  gem.email       = 'cmb211@cam.ac.uk'
+  gem.files       = `git ls-files`.split("\n")
+  gem.executables = ["crb-blast"]
+  gem.require_paths = %w( lib )
+  gem.homepage    = 'http://rubygems.org/gems/crb-blast'
+  gem.license     = 'MIT'
+
+  gem.add_dependency 'trollop', '~> 2.0'
+  gem.add_dependency 'bio', '~> 1.4', '>= 1.4.3'
+  gem.add_dependency 'which', '0.0.2'
+  gem.add_dependency 'threach', '~> 0.2', '>= 0.2.0'
+  gem.add_dependency 'bindeps', '~> 0.0', '>= 0.0.7'
+
+  gem.add_development_dependency 'rake', '~> 10.3', '>= 10.3.2'
+  gem.add_development_dependency 'turn', '~> 0.9', '>= 0.9.7'
+  gem.add_development_dependency 'simplecov', '~> 0.8', '>= 0.8.2'
+  gem.add_development_dependency 'shoulda-context', '~> 1.2', '>= 1.2.1'
+  gem.add_development_dependency 'coveralls', '~> 0.7'
+end

data/deps/deps.yaml

@@ -0,0 +1,27 @@
+blastplus:
+  binaries:
+    - makeblastdb
+    - blastn
+    - tblastn
+    - blastp
+    - blastx
+    - tblastx
+    - makembindex
+    - psiblast
+    - rpsblast
+    - blastdbcmd
+    - segmasker
+    - dustmasker
+    - blast_formatter
+    - windowmasker
+    - blastdb_aliastool
+    - deltablast
+    - rpstblastn
+    - blastdbcheck
+  version:
+    number: '2.2.29'
+    command: 'blastx -version'
+  url:
+    64bit:
+      macosx: ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.2.29/ncbi-blast-2.2.29+-universal-macosx.tar.gz
+      linux: ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.2.29/ncbi-blast-2.2.29+-x64-linux.tar.gz

data/lib/crb-blast.rb

@@ -1,447 +1,3 @@
-#!/usr/bin/env ruby
-
-require 'bio'
-require 'which'
-require 'hit'
-require 'threach'
-
-class Bio::FastaFormat
-  def isNucl?
-    Bio::Sequence.guess(self.seq, 0.9, 500) == Bio::Sequence::NA
-  end
-
-  def isProt?
-    Bio::Sequence.guess(self.seq, 0.9, 500) == Bio::Sequence::AA
-  end
-end
-
-class CRB_Blast
-
-  include Which
-
-  attr_accessor :query_name, :target_name, :reciprocals
-  attr_accessor :missed
-  attr_accessor :target_is_prot, :query_is_prot
-  attr_accessor :query_results, :target_results, :working_dir
-
-  def initialize query, target, output=nil
-    @query = query
-    @target = target
-    if output.nil?
-      #@working_dir = File.expand_path(File.dirname(query)) # no trailing /
-      @working_dir = "."
-    else
-      @working_dir = File.expand_path(output)
-      mkcmd = "mkdir #{@working_dir}"
-      if !Dir.exist?(@working_dir)
-        puts mkcmd
-        `#{mkcmd}`
-      end
-    end
-    @makedb_path = which('makeblastdb')
-    raise 'makeblastdb was not in the PATH' if @makedb_path.empty?
-    @blastn_path = which('blastn')
-    raise 'blastn was not in the PATH' if @blastn_path.empty?
-    @tblastn_path = which('tblastn')
-    raise 'tblastn was not in the PATH' if @tblastn_path.empty?
-    @blastx_path = which('blastx')
-    raise 'blastx was not in the PATH' if @blastx_path.empty?
-    @blastp_path = which('blastp')
-    raise 'blastp was not in the PATH' if @blastp_path.empty?
-    @makedb_path = @makedb_path.first
-    @blastn_path = @blastn_path.first
-    @tblastn_path = @tblastn_path.first
-    @blastx_path = @blastx_path.first
-    @blastp_path = @blastp_path.first
-  end
-
-  #
-  # makes a blast database from the query and the target
-  #
-  def makedb
-    # only scan the first few hundred entries
-    n = 100
-    # check if the query is a nucl or prot seq
-    query_file = Bio::FastaFormat.open(@query)
-    count_p=0
-    count=0
-    query_file.take(n).each do |entry|
-      count_p += 1 if entry.isProt?
-      count += 1
-    end
-    if count_p > count*0.9
-      @query_is_prot = true
-    else
-      @query_is_prot = false
-    end
-
-    # check if the target is a nucl or prot seq
-    target_file = Bio::FastaFormat.open(@target)
-    count_p=0
-    count=0
-    target_file.take(n).each do |entry|
-      count_p += 1 if entry.isProt?
-      count += 1
-    end
-    if count_p > count*0.9
-      @target_is_prot = true
-    else
-      @target_is_prot = false
-    end
-    # construct the output database names
-    @query_name = File.basename(@query).split('.')[0..-2].join('.')
-    @target_name = File.basename(@target).split('.')[0..-2].join('.')
-
-    # check if the databases already exist in @working_dir
-    make_query_db_cmd = "#{@makedb_path} -in #{@query}"
-    make_query_db_cmd << " -dbtype nucl " if !@query_is_prot
-    make_query_db_cmd << " -dbtype prot " if @query_is_prot
-    make_query_db_cmd << " -title #{query_name} "
-    make_query_db_cmd << " -out #{@working_dir}/#{query_name}"
-    db_query = "#{query_name}.nsq" if !@query_is_prot
-    db_query = "#{query_name}.psq" if @query_is_prot
-    if !File.exists?("#{@working_dir}/#{db_query}")
-      `#{make_query_db_cmd}`
-    end
-
-    make_target_db_cmd = "#{@makedb_path} -in #{@target}"
-    make_target_db_cmd << " -dbtype nucl " if !@target_is_prot
-    make_target_db_cmd << " -dbtype prot " if @target_is_prot
-    make_target_db_cmd << " -title #{target_name} "
-    make_target_db_cmd << " -out #{@working_dir}/#{target_name}"
-
-    db_target = "#{target_name}.nsq" if !@target_is_prot
-    db_target = "#{target_name}.psq" if @target_is_prot
-    if !File.exists?("#{@working_dir}/#{db_target}")
-      `#{make_target_db_cmd}`
-    end
-    @databases = true
-    [@query_name, @target_name]
-  end
-
-  def run_blast(evalue, threads, split)
-    if @databases
-      @output1 = "#{@working_dir}/#{query_name}_into_#{target_name}.1.blast"
-      @output2 = "#{@working_dir}/#{target_name}_into_#{query_name}.2.blast"
-      if @query_is_prot
-        if @target_is_prot
-          bin1 = "#{@blastp_path} "
-          bin2 = "#{@blastp_path} "
-        else
-          bin1 = "#{@tblastn_path} "
-          bin2 = "#{@blastx_path} "
-        end
-      else
-        if @target_is_prot
-          bin1 = "#{@blastx_path} "
-          bin2 = "#{@tblastn_path} "
-        else
-          bin1 = "#{@blastn_path} "
-          bin2 = "#{@blastn_path} "
-        end
-      end
-      if split and threads > 1
-        run_blast_with_splitting evalue, threads, bin1, bin2
-      else
-        run_blast_with_threads evalue, threads, bin1, bin2
-      end
-      return true
-    else
-      return false
-    end
-  end
-
-  def run_blast_with_threads evalue, threads, bin1, bin2
-    # puts "running blast with #{threads} threads"
-    cmd1 = "#{bin1} -query #{@query} -db #{@working_dir}/#{@target_name} "
-    cmd1 << " -out #{@output1} -evalue #{evalue} "
-    cmd1 << " -outfmt \"6 std qlen slen\" "
-    cmd1 << " -max_target_seqs 50 "
-    cmd1 << " -num_threads #{threads}"
-
-    cmd2 = "#{bin2} -query #{@target} -db #{@working_dir}/#{@query_name} "
-    cmd2 << " -out #{@output2} -evalue #{evalue} "
-    cmd2 << " -outfmt \"6 std qlen slen\" "
-    cmd2 << " -max_target_seqs 50 "
-    cmd2 << " -num_threads #{threads}"
-    if !File.exist?("#{@output1}")
-      `#{cmd1}`
-    end
-
-    if !File.exist?("#{@output2}")
-      `#{cmd2}`
-    end
-  end
-
-  def run_blast_with_splitting evalue, threads, bin1, bin2
-    # puts "running blast by splitting input into #{threads} pieces"
-    blasts=[]
-    files = split_input(@query, threads)
-    files.threach(threads) do |thread|
-      cmd1 = "#{bin1} -query #{thread} -db #{@working_dir}/#{@target_name} "
-      cmd1 << " -out #{thread}.blast -evalue #{evalue} "
-      cmd1 << " -outfmt \"6 std qlen slen\" "
-      cmd1 << " -max_target_seqs 50 "
-      cmd1 << " -num_threads 1"
-      if !File.exists?("#{thread}.blast")
-        `#{cmd1}`
-      end
-      blasts << "#{thread}.blast"
-    end
-    cat_cmd = "cat "
-    cat_cmd << blasts.join(" ")
-    cat_cmd << " > #{@output1}"
-    `#{cat_cmd}`
-    files.each do |file|
-      File.delete(file) if File.exist?(file)
-    end
-    blasts.each do |b|
-      File.delete(b) # delete intermediate blast output files
-    end
-
-    blasts=[]
-    files = split_input(@target, threads)
-    files.threach(threads) do |thread|
-      cmd2 = "#{bin2} -query #{thread} -db #{@working_dir}/#{@query_name} "
-      cmd2 << " -out #{thread}.blast -evalue #{evalue} "
-      cmd2 << " -outfmt \"6 std qlen slen\" "
-      cmd2 << " -max_target_seqs 50 "
-      cmd2 << " -num_threads 1"
-      if !File.exists?("#{thread}.blast")
-        `#{cmd2}`
-      end
-      blasts << "#{thread}.blast"
-    end
-    cat_cmd = "cat "
-    cat_cmd << blasts.join(" ")
-    cat_cmd << " > #{@output2}"
-    `#{cat_cmd}`
-    files.each do |file|
-      File.delete(file) if File.exist?(file)
-    end
-    blasts.each do |b|
-      File.delete(b) # delete intermediate blast output files
-    end
-
-  end
-
-  def split_input filename, pieces
-    input = {}
-    name = nil
-    seq=""
-    File.open(filename).each_line do |line|
-      if line =~ /^>(.*)$/
-        if name
-          input[name]=seq
-          seq=""
-        end
-        name = $1
-      else
-        seq << line.chomp
-      end
-    end
-    input[name]=seq
-    # construct list of output file handles
-    outputs=[]
-    output_files=[]
-    pieces.times do |n|
-      outfile = "#{filename}_chunk_#{n}.fasta"
-      outfile = File.expand_path(outfile)
-      outputs[n] = File.open("#{outfile}", "w")
-      output_files[n] = "#{outfile}"
-    end
-    # write sequences
-    count=0
-    input.each_pair do |name, seq|
-      outputs[count].write(">#{name}\n")
-      outputs[count].write("#{seq}\n")
-      count += 1
-      count %= pieces
-    end
-    outputs.each do |out|
-      out.close
-    end
-    output_files
-  end
-
-  def load_outputs
-    if File.exist?("#{@working_dir}/reciprocal_hits.txt")
-      # puts "reciprocal output already exists"
-    else
-      @query_results = Hash.new
-      @target_results = Hash.new
-      q_count=0
-      t_count=0
-      if !File.exists?("#{@output1}")
-        raise RuntimeError.new("can't find #{@output1}")
-      end
-      if !File.exists?("#{@output2}")
-        raise RuntimeError.new("can't find #{@output2}")
-      end
-      if File.exists?("#{@output1}") and File.exists?("#{@output2}")
-        File.open("#{@output1}").each_line do |line|
-          cols = line.chomp.split("\t")
-          hit = Hit.new(cols)
-          @query_results[hit.query] = [] if !@query_results.has_key?(hit.query)
-          @query_results[hit.query] << hit
-          q_count += 1
-        end
-        File.open("#{@output2}").each_line do |line|
-          cols = line.chomp.split("\t")
-          hit = Hit.new(cols)
-          @target_results[hit.query] = [] if !@target_results.has_key?(hit.query)
-          @target_results[hit.query] << hit
-          t_count += 1
-        end
-      else
-        raise "need to run blast first"
-      end
-    end
-    [q_count, t_count]
-  end
-
-  # fills @reciprocals with strict reciprocal hits from the blast results
-  def find_reciprocals
-    if File.exist?("#{@working_dir}/reciprocal_hits.txt")
-      # puts "reciprocal output already exists"
-    else
-      @reciprocals = Hash.new
-      @missed = Hash.new
-      @evalues = []
-      @longest = 0
-      hits = 0
-      @query_results.each_pair do |query_id, list_of_hits|
-        list_of_hits.each_with_index do |target_hit, query_index|
-          if @target_results.has_key?(target_hit.target)
-            list_of_hits_2 = @target_results[target_hit.target]
-            list_of_hits_2.each_with_index do |query_hit2, target_index|
-              if query_index == 0 && target_index == 0 &&
-                 query_id == query_hit2.target
-                e = target_hit.evalue.to_f
-                e = 1e-200 if e==0
-                e = -Math.log10(e)
-                if !@reciprocals.key?(query_id)
-                  @reciprocals[query_id] = []
-                end
-                @reciprocals[query_id] << target_hit
-                hits += 1
-                @longest = target_hit.alnlen  if target_hit.alnlen > @longest
-                @evalues << {:e => e, :length => target_hit.alnlen}
-              elsif query_id == query_hit2.target
-                if !@missed.key?(query_id)
-                  @missed[query_id] = []
-                end
-                @missed[query_id] << target_hit
-              end
-            end
-          end
-        end
-      end
-    end
-    return hits
-  end
-
-  def find_secondaries
-
-    if File.exist?("#{@working_dir}/reciprocal_hits.txt")
-      # puts "reciprocal output already exists"
-    else
-      length_hash = Hash.new
-      fitting = Hash.new
-      @evalues.each do |h|
-        length_hash[h[:length]] = [] if !length_hash.key?(h[:length])
-        length_hash[h[:length]] << h
-      end
-
-      (10..@longest).each do |centre|
-        e = 0
-        count = 0
-        s = centre*0.1
-        s = s.to_i
-        s = 5 if s < 5
-        (-s..s).each do |side|
-          if length_hash.has_key?(centre+side)
-            length_hash[centre+side].each do |point|
-              e += point[:e]
-              count += 1
-            end
-          end
-        end
-        if count>0
-          mean = e/count
-          fitting[centre] = mean
-        end
-      end
-      hits = 0
-      @missed.each_pair do |id, list|
-        list.each do |hit|
-          l = hit.alnlen.to_i
-          e = hit.evalue
-          e = 1e-200 if e==0
-          e = -Math.log10(e)
-          if fitting.has_key?(l)
-            if e >= fitting[l]
-              if !@reciprocals.key?(id)
-                @reciprocals[id] = []
-                found=false
-                @reciprocals[id].each do |existing_hit|
-                  if existing_hit.query == hit.query &&
-                    existing_hit.target == hit.target
-                   found=true
-                  end
-                end
-                if !found
-                  @reciprocals[id] << hit
-                  hits += 1
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-    return hits
-  end
-
-  def clear_memory
-    # running lots of jobs at the same time was keeping a lot of stuff in
-    # memory that you might not want so this empties out those big hashes.
-    @query_results = nil
-    @target_results = nil
-  end
-
-  def run evalue, threads, split
-    makedb
-    run_blast evalue, threads, split
-    load_outputs
-    find_reciprocals
-    find_secondaries
-  end
-
-  def size
-    hits=0
-    @reciprocals.each_pair do |key, list|
-      list.each do |hit|
-        hits += 1
-      end
-    end
-    hits
-  end
-
-  def write_output
-    s=""
-    unless @reciprocals.nil?
-      @reciprocals.each_pair do |query_id, hits|
-        hits.each do |hit|
-          s << "#{hit}\n"
-        end
-      end
-      File.open("#{@working_dir}/reciprocal_hits.txt", "w") {|f| f.write s }
-    end
-  end
-
-  def has_reciprocal? contig
-    return true if @reciprocals.has_key?(contig)
-    return false
-  end
-end
+require 'crb-blast/cmd'
+require 'crb-blast/hit'
+require 'crb-blast/crb-blast'