transrate 0.1.0 → 0.2.0

data/deps/deps.yaml

@@ -0,0 +1,55 @@
+blastplus:
+  binaries:
+    - makeblastdb
+    - blastn
+    - tblastn
+    - blastp
+    - blastx
+    - tblastx
+    - makembindex
+    - psiblast
+    - rpsblast
+    - blastdbcmd
+    - segmasker
+    - dustmasker
+    - blast_formatter
+    - windowmasker
+    - blastdb_aliastool
+    - deltablast
+    - rpstblastn
+    - blastdbcheck
+  version:
+    number: '2.2.29'
+    command: 'blastx -version'
+  url:
+    64bit:
+      macosx: ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.2.29/ncbi-blast-2.2.29+-universal-macosx.tar.gz
+      linux: ftp://ftp.ncbi.nlm.nih.gov/blast/executables/blast+/2.2.29/ncbi-blast-2.2.29+-x64-linux.tar.gz
+bowtie2:
+  binaries:
+    - bowtie2
+    - bowtie2-align-l
+    - bowtie2-align-s
+    - bowtie2-build
+    - bowtie2-build-l
+    - bowtie2-build-s
+    - bowtie2-inspect
+    - bowtie2-inspect-l
+    - bowtie2-inspect-s
+  version:
+    number: '2.2.3'
+    command: 'bowtie2 --version'
+  url:
+    64bit:
+      linux: http://downloads.sourceforge.net/project/bowtie-bio/bowtie2/2.2.3/bowtie2-2.2.3-linux-x86_64.zip
+      macosx: http://downloads.sourceforge.net/project/bowtie-bio/bowtie2/2.2.3/bowtie2-2.2.3-macos-x86_64.zip
+express:
+  binaries:
+    - express
+  version:
+    number: '1.5.1'
+    command: 'express --version'
+  url:
+    64bit:
+      linux: http://bio.math.berkeley.edu/eXpress/downloads/express-1.5.1/express-1.5.1-linux_x86_64.tgz
+      macosx: http://bio.math.berkeley.edu/eXpress/downloads/express-1.5.1/express-1.5.1-macosx_x86_64.tgz

data/lib/transrate.rb

@@ -1,18 +1,33 @@
+# before the
+require 'rbconfig'
+require 'yell'
+RbConfig::CONFIG['CFLAGS'] = ''
+
 require 'transrate/transrater'
 require 'transrate/version'
+require 'transrate/contig'
 require 'transrate/assembly'
 require 'transrate/bowtie2'
 require 'transrate/read_metrics'
-require 'transrate/usearch'
-require 'transrate/rb_hit'
-require 'transrate/reciprocal_annotation'
 require 'transrate/comparative_metrics'
+require 'transrate/contig_metrics'
 require 'transrate/metric'
 require 'transrate/dimension_reduce'
-require 'transrate/express'
+require 'transrate/samtools'
+require 'transrate/cmd'
 
 # Transrate is a comprehensive transcriptome assembly
 # quality assessment tool.
 module Transrate
 
+  # Create the universal logger and include it in Object
+  # making the logger object available everywhere
+  Yell.new(:format => "[%5L]: %m") do |l|
+    l.level = :info
+    l.name = Object
+    l.adapter STDOUT, level: [:debug, :info, :warn]
+    l.adapter STDERR, level: [:error, :fatal]
+  end
+  Object.send :include, Yell::Loggable
+
 end # Transrate

data/lib/transrate/assembly.rb

@@ -1,8 +1,6 @@
 require 'bio'
-require 'bettersam'
 require 'csv'
 require 'forwardable'
-require 'inline'
 
 module Transrate
 
@@ -30,36 +28,28 @@ module Transrate
     extend Forwardable
     def_delegators :@assembly, :each, :<<, :size, :length
 
-    attr_accessor :ublast_db
-    attr_accessor :orfs_ublast_db
+    attr_accessor :file
     attr_reader :assembly
     attr_reader :has_run
-    attr_writer :n_bases
-    attr_accessor :file
+    attr_accessor :n_bases
     attr_reader :n50
+    attr_accessor :contig_metrics
 
     # Create a new Assembly.
     #
     # @param file [String] path to the assembly FASTA file
     def initialize file
-      @file = file
+      @file = File.expand_path file
+      unless File.exist? @file
+        raise IOError.new "Assembly file doesn't exist: #{@file}"
+      end
       @assembly = []
       @n_bases = 0
       Bio::FastaFormat.open(file).each do |entry|
         @n_bases += entry.length
-        @assembly << entry
+        @assembly << Contig.new(entry)
       end
-    end
-
-    # Return basic statistics about the assembly in 
-    # the specified FASTA file
-    #
-    # @param file [String] path to assebmly FASTA file
-    #
-    # @return [Hash] basic statistics about the assembly
-    def self.stats_from_fasta file
-      a = Assembly.new file
-      a.basic_stats
+      @contig_metrics = ContigMetrics.new self
     end
 
     # Generate and store the basic statistics for this assembly
@@ -74,6 +64,7 @@ module Transrate
         singleton_class.class_eval { attr_accessor attr_ivar }
         self.instance_variable_set(ivar, value)
       end
+      @contig_metrics.run
       @has_run = true
     end
 
@@ -85,60 +76,14 @@ module Transrate
     # @param threads [Integer] number of threads to use
     #
     # @return [Hash] basic statistics about the assembly
-    def basic_stats threads=8
-      
-      # disable threading basic stats for now
-      threads = 1
-
-      # create a work queue to process contigs in parallel
-      queue = Queue.new
-      
-      # split the contigs into equal sized bins, one bin per thread
-      binsize = (@assembly.size / threads.to_f).ceil
-      @assembly.each_slice(binsize) do |bin|
-        queue << bin
-      end
-
-      # a classic threadpool - an Array of threads that allows
-      # us to assign work to each thread and then aggregate their
-      # results when they are all finished
-      threadpool = []
-
-      # assign one bin of contigs to each thread from the queue.
-      # each thread will process its bin of contigs and then wait
-      # for the others to finish.
-      semaphore = Mutex.new
-      stats = []
-
-      threads.times do
-        threadpool << Thread.new do |thread|
-          # keep looping until we run out of bins
-          until queue.empty?
-
-            # use non-blocking pop, so an exception is raised
-            # when the queue runs dry
-            bin = queue.pop(true) rescue nil
-            if bin
-              # calculate basic stats for the bin, storing them
-              # in the current thread so they can be collected
-              # in the main thread.
-              bin_stats = basic_bin_stats bin
-              semaphore.synchronize { stats << bin_stats }
-            end
-          end
-        end
-      end
-
-      # collect the stats calculated in each thread and join
-      # the threads to terminate them
-      threadpool.each(&:join)
-
-      # merge the collected stats and return then
-      merge_basic_stats stats
-
+    def basic_stats threads=1
+      return @basic_stats if @basic_stats
+      bin = @assembly.dup
+      @basic_stats = basic_bin_stats bin
+      @basic_stats
     end # basic_stats
 
-    
+
     # Calculate basic statistics in an single thread for a bin
     # of contigs.
     #
@@ -156,13 +101,13 @@ module Transrate
     #
     # @param [Array] bin An array of Bio::Sequence objects
     # representing contigs in the assembly
-    
-    def basic_bin_stats bin
 
+    def basic_bin_stats bin
+      
       # cumulative length is a float so we can divide it
       # accurately later to get the mean length
       cumulative_length = 0.0
-      
+
       # we'll calculate Nx for x in [10, 30, 50, 70, 90]
       # to do this we create a stack of the x values and
       # pop the first one to set the first cutoff. when
@@ -174,23 +119,30 @@ module Transrate
       x2 = x.clone
       cutoff = x2.pop / 100.0
       res = []
-      n1k = 0
-      n10k = 0
-      orf_length_sum = 0
-
+      n_under_200, n_over_1k, n_over_10k, n_with_orf, orf_length_sum = 0,0,0,0,0
       # sort the contigs in ascending length order
       # and iterate over them
-      bin.sort_by! { |c| c.seq.size }
+      bin.sort_by! { |c| c.seq.length }
       bin.each do |contig|
-        
+
         # increment our long contig counters if this
         # contig is above the thresholds
-        n1k += 1 if contig.length > 1_000
-        n10k += 1 if contig.length > 10_000
+        if contig.length < 200
+          # ignore contigs less than 200 bases,
+          # but record how many there are
+          n_under_200 += 1
+          next
+        end
+        n_over_1k += 1 if contig.length > 1_000
+        n_over_10k += 1 if contig.length > 10_000
 
         # add the length of the longest orf to the
         # running total
-        orf_length_sum += orf_length(contig.seq)
+        orf_length = contig.orf_length
+        orf_length_sum += orf_length
+        # only consider orfs that are realistic length
+        # (here we set minimum amino acid length as 50)
+        n_with_orf += 1 if orf_length > 149
 
         # increment the cumulative length and check whether the Nx
         # cutoff has been reached. if it has, store the Nx value and
@@ -199,125 +151,84 @@ module Transrate
         if cumulative_length >= @n_bases * cutoff
           res << contig.length
           if x2.empty?
-            cutoff=1
+            cutoff = 1
           else
             cutoff = x2.pop / 100.0
-          end 
+          end
         end
 
       end
 
+      # if there aren't enough sequences we might have no value for some
+      # of the Nx. Fill the empty ones in with the longest contig length.
+      while res.length < x.length do
+        res << bin.last.length
+      end
+
       # calculate and return the statistics as a hash
       mean = cumulative_length / @assembly.size
-      ns = Hash[x.map { |n| "N#{n}" }.zip(res)]
+      ns = Hash[x.map { |n| "n#{n}" }.zip(res)]
       {
         'n_seqs' => bin.size,
         'smallest' => bin.first.length,
         'largest' => bin.last.length,
         'n_bases' => n_bases,
         'mean_len' => mean,
-        'n_1k' => n1k,
-        'n_10k' => n10k,
-        'orf_percent' => 300 * orf_length_sum / (@assembly.size * mean)
+        'n_under_200' => n_under_200,
+        'n_over_1k' => n_over_1k,
+        'n_over_10k' => n_over_10k,
+        'n_with_orf' => n_with_orf,
+        'mean_orf_percent' => 300 * orf_length_sum / (@assembly.size * mean)
       }.merge ns
 
     end # basic_bin_stats
 
-    def merge_basic_stats stats
-      # convert the array of hashes into a hash of arrays
-      collect = Hash.new{|h,k| h[k]=[]}
-      stats.each_with_object(collect) do |collect, result|
-        collect.each{ |k, v| result[k] << v }
-      end
-      merged = {}
-      collect.each_pair do |stat, values|
-        if stat == 'orf_percent'  || /N[0-9]{2}/ =~ stat
-          # store the mean
-          merged[stat] = values.inject(:+) / values.size
-        elsif stat == 'smallest'
-          merged[stat] = values.min
-        elsif stat == 'largest'
-          merged[stat] = values.max
-        else
-          # store the sum
-          merged[stat] = values.inject(:+)
+    # Calls *block* with two arguments, the contig and an array
+    # of integer per-base coverage counts.
+    #
+    # @param bam [Bio::Db::Sam] a bam alignment of reads against this assembly
+    # @param block [Block] the block to call
+    def each_with_coverage(bam, &block)
+      logger.debug 'enumerating assembly with coverage'
+      # generate coverage with samtools
+      covfile = Samtools.coverage bam
+      # get an assembly enumerator
+      assembly_enum = @assembly.to_enum
+      contig = assembly_enum.next
+      # precreate an array of the correct size to contain
+      # coverage. this is necessary because samtools mpileup
+      # doesn't print a result line for bases with 0 coverage
+      contig.coverage = Array.new(contig.length, 0)
+      # the columns we need
+      name_i, pos_i, cov_i = 0, 1, 3
+      # parse the coverage file
+      File.open(covfile).each_line do |line|
+        cols = line.chomp.split("\t")
+        unless (cols && cols.length > 4)
+          # last line
+          break
         end
+        # extract the columns
+        name, pos, cov = cols[name_i], cols[pos_i].to_i, cols[cov_i].to_i
+        unless contig.name == name
+          while contig.name != name
+            begin
+              block.call(contig, contig.coverage)
+              contig = assembly_enum.next
+              contig.coverage = Array.new(contig.length, 0)
+            rescue StopIteration => stop_error
+              logger.error 'reached the end of assembly enumerator while ' +
+                        'there were contigs left in the coverage results'
+              logger.error "final assembly contig: #{@assembly.last.name}"
+              logger.error "coverage contig: #{name}"
+              raise stop_error
+            end
+          end
+        end
+        contig.coverage[pos - 1] = cov
       end
-
-      merged
-      
-    end # merge_basic_stats
-     
-    inline do |builder|
-
-      builder.c <<SRC
-        static
-        void
-        longest_orf(VALUE _s) {
-          int i,sl,longest=0;
-          int len[6];
-          char * c_str;
-
-          sl = RSTRING_LEN(_s);
-          c_str = StringValueCStr(_s);
-          for (i=0;i<6;i++) {
-            len[i]=0;
-          }
-          for (i=0;i<sl-2;i++) {
-            if (c_str[i]=='T' &&
-              ((c_str[i+1]=='A' && c_str[i+2]=='G') ||
-              (c_str[i+1]=='A' && c_str[i+2]=='A') ||
-              (c_str[i+1]=='G' && c_str[i+2]=='A'))) { 
-              if (len[i%3] > longest) {
-                longest = len[i%3];
-              }
-              len[i%3]=0;
-            } else {
-              len[i%3]++;
-            }
-            if (c_str[i+2]=='A' &&
-              ((c_str[i]=='C' && c_str[i+1]=='T') ||
-              (c_str[i]=='T' && c_str[i+1]=='T') ||
-              (c_str[i]=='T' && c_str[i+1]=='C'))) { 
-              if (len[3+i%3] > longest) {
-                longest = len[3+i%3];
-              }
-              len[3+i%3]=0;
-            } else {
-              len[3+i%3]++;
-            }
-          }
-          if (len[i%3] > longest) {
-            longest = len[i%3];
-          }
-          if (len[3+i%3] > longest) {
-            longest = len[3+i%3];
-          }
-          return INT2NUM(longest);
-        }
-SRC
-    end
-
-    # finds longest orf in a sequence
-    def orf_length sequence
-      longest = longest_orf(sequence)
-      return longest
-    end
-
-    # return the number of bases in the assembly, calculating
-    # from the assembly if it hasn't already been done.
-    def n_bases
-      unless @n_bases
-        @n_bases = 0
-        @assembly.each { |s| @n_bases += s.length }
-      end
-      @n_bases
-    end
-
-    def print_stats
-      self.basic_stats.map do |k, v| 
-        "#{k}#{" " * (20 - (k.length + v.to_i.to_s.length))}#{v.to_i}"
-      end.join('\n')
+      # yield the final contig
+      block.call(contig, contig.coverage)
     end
 
   end # Assembly