transrate 0.3.1 → 1.0.0.alpha.1

data/lib/transrate/samtools.rb

@@ -1,20 +1,31 @@
-require 'bio-samtools'
-
 module Transrate
 
   class Samtools
 
+    class SamtoolsError < StandardError; end
+
     # Get the path to the samtools binary built when bio-samtools
     # was installed
     def self.path
-      gem_path = Gem.loaded_specs['bio-samtools'].full_gem_path
-      return File.join(gem_path, 'lib/bio/db/sam/external/samtools')
+      if !@path
+        which_samtools = Cmd.new("which samtools")
+        which_samtools.run
+        if !which_samtools.status.success?
+          raise SamtoolsError.new("could not find samtools in the path")
+        end
+        @path = which_samtools.stdout.split("\n").first
+      end
+      return @path
     end
 
     # Run a samtools command
     def self.run cmd
       runcmd = Cmd.new "#{Samtools.path} #{cmd}"
       runcmd.run
+      if !runcmd.status.success?
+        logger.warn "Samtools command failed: #{runcmd}" +
+                    "\n#{runcmd.stderr}"
+      end
       runcmd.stdout
     end
 
@@ -22,20 +33,45 @@ module Transrate
     def self.sam_to_bam samfile
       bamfile = File.basename(samfile, '.sam') + '.bam'
       bamfile = File.expand_path bamfile
-      Samtools.run "view -bS #{File.expand_path samfile} > #{bamfile}"
-      File.expand_path bamfile
+      if !File.exist?(bamfile)
+        Samtools.run "view -bS #{File.expand_path samfile} > #{bamfile}"
+      end
+      bamfile
     end
 
     # Sort a bam file, returning the path to the sorted bamfile
-    def self.sort_bam bamfile
+    def self.sort_bam bamfile, threads=4
+      # the sort command behaves inconsistently with the other commands:
+      # it takes an output prefix rather than a filename
+      # and automatically adds the .bam extension
+      sorted = File.basename(bamfile, '.bam') + '.sorted'
+      if !File.exist?("#{sorted}.bam")
+        cmd = "sort"
+        cmd << " -l #{threads}"
+        cmd << " #{File.expand_path bamfile} #{sorted}"
+        Samtools.run cmd
+      end
+      File.expand_path(sorted + '.bam')
+    end
+
+    # Sort a bam file by readname only, returning the path to th
+    # sorted bamfile
+    def self.readsort_bam bamfile, threads=4
       # the sort command behaves inconsistently with the other commands:
       # it takes an output prefix rather than a filename
       # and automatically adds the .bam extension
       sorted = File.basename(bamfile, '.bam') + '.sorted'
-      Samtools.run "sort #{File.expand_path bamfile} #{sorted}"
+      if !File.exist?("#{sorted}.bam")
+        cmd = "sort"
+        cmd << " -l #{threads}"
+        cmd << " -n" # sort by read name only
+        cmd << " #{File.expand_path bamfile} #{sorted}"
+        Samtools.run cmd
+      end
       File.expand_path(sorted + '.bam')
     end
 
+
     # Index a bamfile, returning the path to the index
     def self.index_bam bamfile
       index = File.basename(bamfile, '.bam') + '.bai'
@@ -57,17 +93,53 @@ module Transrate
     # return the path to the coverage file
     def self.coverage bam
       outfile = File.expand_path "#{File.basename(bam.fasta)}.coverage"
-      cmd = "mpileup"
-      cmd += " -f #{File.expand_path bam.fasta}" # reference
-      cmd += " -B" # don't calculate BAQ quality scores
-      cmd += " -Q0" # include all reads ignoring quality
-      cmd += " -I" # don't do genotype calculations
-      cmd += " #{File.expand_path bam.bam}" # the bam file
-      cmd += " > #{outfile}"
-      Samtools.run cmd
+      if !File.exist?(outfile)
+        cmd = "mpileup"
+        cmd += " -f #{File.expand_path bam.fasta}" # reference
+        cmd += " -B" # don't calculate BAQ quality scores
+        cmd += " -Q0" # include all reads ignoring quality
+        cmd += " -I" # don't do genotype calculations
+        cmd += " #{File.expand_path bam.bam}" # the bam file
+        cmd += " > #{outfile}"
+        Samtools.run cmd
+      end
+      outfile
+    end
+
+    # Calculate per-base coverage and mapQ score from a sorted, indexed
+    # bam file. Return the path to the coverage file.
+    def self.bam_to_bcf(bam, fasta)
+      outfile = File.expand_path "#{File.basename(fasta)}.bcf"
+      if !File.exist?(outfile)
+        cmd = "samtools mpileup"
+        cmd << " -f #{File.expand_path fasta}" # reference
+        cmd << " -B" # don't calculate BAQ quality scores
+        cmd << " -q0" # include all multimapping reads
+        cmd << " -Q0" # include all reads ignoring quality
+        cmd << " -I" # don't do genotype calculations
+        cmd << " -u" # output uncompressed bcf format
+        cmd << " #{File.expand_path bam}" # the bam file
+        cmd << " | bcftools view -cg - "
+        cmd << " > #{outfile}"
+        mpileup = Cmd.new cmd
+        mpileup.run
+        if !mpileup.status.success?
+          raise RuntimeError.new("samtools and bcftools failed")
+        end
+      end
       outfile
     end
 
+    def self.merge_bam left, right, out, threads=1
+      cmd = "merge"
+      cmd << " -@ #{threads}"
+      cmd << " #{out}"
+      cmd << " #{left}"
+      cmd << " #{right}"
+      Samtools.run cmd
+      out
+    end
+
   end
 
 end

data/lib/transrate/snap.rb

@@ -0,0 +1,123 @@
+module Transrate
+
+  class SnapError < StandardError
+  end
+
+  class Snap
+
+    attr_reader :index_name, :sam, :read_count
+
+    def initialize
+      which_snap = Cmd.new('which snap')
+      which_snap.run
+      if !which_snap.status.success?
+        raise SnapError.new("could not find snap in the path")
+      end
+      @snap = which_snap.stdout.split("\n").first
+
+      @index_built = false
+      @index_name = ""
+    end
+
+    def build_paired_cmd l, r, threads
+      cmd = "#{@snap} paired #{@index_name}"
+      l.split(",").zip(r.split(",")).each do |left, right|
+        cmd << " #{left} #{right}"
+      end
+      # NOTE: do NOT turn on the -so flag (sort bam output)
+      # it violates the basic assumption of eXpress's streaming
+      # algorithm: that the fragments are observed in approximately
+      # random order.
+      cmd << " -o #{@bam}"
+      cmd << " -s 0 1000" # min and max distance between paired-read starts
+      cmd << " -H 300000" # max seed hits to consider in paired mode
+      cmd << " -h 2000" # max seed hits to consider when reverting to single
+      cmd << " -I"   # ignore read IDs
+      cmd << " -d 30" # max edit distance (function of read length?)
+      cmd << " -t #{threads}"
+      cmd << " -b" # bind threads to cores
+      cmd << " -M"  # format cigar string
+      cmd << " -sa" # keep all alignments, don't discard 0x100
+      # cmd << " -C++" # trim low-quality bases from front and back of reads
+      cmd
+    end
+
+    def map_reads(file, left, right, insertsize: 200,
+                  insertsd: 50, outputname: nil, threads: 8)
+      raise SnapError.new("Index not built") if !@index_built
+
+      lbase = File.basename(left.split(",").first)
+      rbase = File.basename(right.split(",").first)
+      index = File.basename(@index_name)
+      @bam = File.expand_path("#{lbase}.#{rbase}.#{index}.bam")
+      @read_count_file = "#{lbase}-#{rbase}-read_count.txt"
+
+      unless File.exists? @bam
+        snapcmd = build_paired_cmd(left, right, threads)
+        runner = Cmd.new snapcmd
+        runner.run
+        save_readcount runner.stdout
+        unless runner.status.success?
+          raise SnapError.new("Snap failed\n#{runner.stderr}")
+        end
+      else
+        load_readcount left
+      end
+      @bam
+    end
+
+    def save_readcount stdout
+      stdout.split("\n").each do |line|
+        cols = line.split(/\s+/)
+        if cols[0]=="2000" and cols[1]=="30"
+          @read_count = cols[8].to_i / 2
+          File.open("#{@read_count_file}", "wb") do |out|
+            out.write("#{@read_count}\n")
+          end
+        end
+      end
+    end
+
+    def load_readcount reads
+      @read_count = 0
+      if File.exist?("#{@read_count_file}")
+        @read_count = File.open("#{@read_count_file}").readlines.join.to_i
+      else
+        reads.split(",").each do |l|
+          cmd = "wc -l #{l}"
+          count = Cmd.new(cmd)
+          count.run
+          if count.status.success?
+            @read_count += count.stdout.strip.split(/\s+/).first.to_i/4
+            File.open("#{@read_count_file}", "wb") do |out|
+              out.write("#{@read_count}\n")
+            end
+          else
+            logger.warn "couldn't get number of reads from #{l}"
+          end
+        end
+      end
+    end
+
+    def build_index file, threads
+      @index_name = File.basename(file, File.extname(file))
+      unless Dir.exists?(@index_name)
+        overflow = 500
+        cmd = "#{@snap} index #{file} #{@index_name}"
+        cmd << " -s 23"
+        cmd << " -t#{threads}"
+        cmd << " -bSpace" # contig name terminates with space char
+        runner = Cmd.new cmd
+        runner.run
+        if !runner.status.success?
+          err = runner.stderr
+          msg = "Failed to build Snap index\n#{runner.stderr}"
+          raise SnapError.new(msg)
+        end
+      end
+      @index_built = true
+    end
+
+  end # Snap
+
+end # Transrate

data/lib/transrate/transrater.rb

@@ -22,10 +22,8 @@ module Transrate
     # @param right [String] path to the right reads
     # @param insertsize [Integer] mean insert size of the read pairs
     # @param insertsd [Integer] standard deviation of the read pair insert size
-    def initialize(assembly, reference,
-                   left: nil, right: nil,
-                   insertsize: nil, insertsd: nil,
-                   threads: 1)
+    def initialize(assembly, reference, left: nil, right: nil,
+                   insertsize: nil, insertsd: nil, threads: 1)
       if assembly
         if assembly.is_a?(Assembly)
           @assembly = assembly
@@ -64,23 +62,22 @@ module Transrate
       comparative_metrics
     end
 
-    # Reduce all metrics for the assembly to a single quality score.
-    #
-    #
-    #
+    # Calculate the geometric mean of an array of numbers
+    def geomean(x)
+      sum = 0.0
+      x.each{ |v| sum += Math.log(v) }
+      sum /= x.size
+      Math.exp(sum)
+    end
+
+    # Reduce all metrics for the assembly to a single quality score
+    # by taking the geometric mean of the scores for all contigs
+    # and multiplying it by the proportion of fragments whose most likely
+    # mapping is consistent with the assembly
     # @return [Integer] the assembly score
     def assembly_score
-      @score, pg, rc = nil
-      if @read_metrics.has_run
-        pg = Metric.new('pg', @read_metrics.pr_good_mapping, 0.0)
-      end
-      if @comparative_metrics.has_run
-        rc = Metric.new('rc', @comparative_metrics.reference_coverage, 0.0)
-      end
-      if (pg && rc)
-        @score = DimensionReduce.dimension_reduce([pg, rc])
-      end
-      return @score
+      @score = geomean assembly.assembly.values.map{ |contig| contig.score }
+      return @score * @read_metrics.p_good_mapping
     end
 
     def assembly_metrics

data/lib/transrate/version.rb

@@ -8,10 +8,10 @@ module Transrate
   # the Gem. Versions must be incremented in accordance with
   # Semantic Versioning 2.0 (http://semver.org/).
   module VERSION
-    MAJOR = 0
-    MINOR = 3
-    PATCH = 1
-    BUILD = nil
+    MAJOR = 1
+    MINOR = 0
+    PATCH = 0
+    BUILD = 'alpha.1'
 
     STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
   end

data/test/data/bridging_reads.l.fastq

@@ -18,3 +18,23 @@ gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg
 ACGACGCGCACCACATCAACAAGCACAACTTCCGTGTCCCCTTCGTCTGTGGCTGCCGTGACCTTGGTGAGGCGCTCCGGAGGGTCCGTGAGGGCGCCGC
 +
 gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg
+@FCC2HFRACXX:7:1101:2015:1981#TGACCAAT/1
+GTCGAGGCCGTCAGGCATGTGCGGTCCGTCATGGGCGATGTCCGTGCGCTCCGGAACATGGATGATGATGAGGTGTTCGCGTATGCTAAGCGCATCGCCG
++
+_[_eeeeegebe`efddgaeehhhhfgfQ^eb`adfgfffhZ_db\adb`aaabaWQT]`b_b]Y_b`cbbb`]`][]^]aaX^bbdbbY^[_a_[[O][
+@FCC2HFRACXX:7:1101:7968:3054#TGACCAAT/1
+GGCCACGCCTGCTGATGCCGCGCTCATGATGCAGCTTGGGTGCGACGGCGTCTTCGTCGGCTC
++
+bbbeeeeef`gcghcegdghfhfdfdfdghhfgfhhhhhgG_Z`__`c[_RZa^baX^aaZ_a
+@FCC2HFRACXX:7:1101:18523:3712#TGACCAAT/1
+CTCATGTCTCTCCATCAGTAATAACGGGACTGAATCAATGGTAAGAAACAAAGCTATGGTACTATGCAGAACCTCTTTATTTTT
++
+__^cc`\cecabcdea_d[`b^[abaUeW^a_e[S^afS^^Y^acbfedeb]^cacefh_V\W\efbddYcd`deV\aRV^ccc
+@FCC2HFRACXX:7:1101:6215:4551#TGACCAAT/1
+TATGATTTGGTGATGCAGACCAAGCAGCTGGGCCGCCTCCCTGTTGTGCAGTTCGCGGCCGGGGGTGTGGCCACGCCTGCTGATGCCGCGCTCATGATGC
++
+__[cccceggbeeddghhdhhhhhdhhfbfhhhhhhhhhhhhhghfefg_fdcgbdaabaaa_[]LEO^^^``[X_^^`ab`]]_bbac_X]XEYR_b]_
+@FCC2HFRACXX:7:1101:9701:4567#TGACCAAT/1
+GTTCAGGCTGTCACCCACTACAGCGACCCTACCATCCTCGCCGACGTCAGCGCCGGACTCGGGGAGGCCATGGTCGGCATCAACCTCAACGACCCTA
++
+b_beeeeeggggfiihhihihifhiiiiiiihfhefhiihiiiiigeedcdcca^cacccacccccc_acbbbS]][_[ab_`X`]`bcc[[X_a^b

data/test/data/bridging_reads.r.fastq

@@ -18,3 +18,23 @@ gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg
 GGACCGGCAACATCGTCGAGGCCGTCAGGCATGTGCGGTCCGTCATGGGCGATGTCCGTGCGCTCCGGAACATGGATGATGATGAGGTGTTCGCGTATGC
 +
 gggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggggg
+@FCC2HFRACXX:7:1101:2015:1981#TGACCAAT/2
+CCCGGCCGCGAACTGCACAACAGGGAGGCGGCCCAGCTGCTTGGTCTGCATCACCAAATCATACGGCGCGGCGATGCGCT
++
+___cccecec`eghfRc`efgfhaeefgiicccc^aX_]Y`bbcYbW`_XX]bbbbccccb`bbaa]_TTXTZTXT^[[_
+@FCC2HFRACXX:7:1101:7968:3054#TGACCAAT/2
+TCGCTGTAGTGGGTGACAGCCTGAACGATGGCACGCGCGCGGCGAGCAGGGTCGCCACTCTTGAAGATTCCCGAGCCGACGAAG
++
+___ea`e`cgegf^b`bbdgfffhdhhfhgheihfhihhiS\[RKWTL[`YX^_acccccccbcccbbc_abcR]]aXT[_]EQ
+@FCC2HFRACXX:7:1101:18523:3712#TGACCAAT/2
+CATAAGCAGGGCTGGCAGAGCGACAGACATACAAACCGTTCAACATTTGATTGAGTACATGATAAACCAGGGGAGCCTAGTCTAACTTAATACAGTAGCT
++
+^_accacagggabbfa]_f`Z_cgfYcZ_[aeggiihaeaeggghgdg_g__gghVb^``d`bdcaced^`__RW^^bbbcbcc]_]]b[]c]_`SY`YY
+@FCC2HFRACXX:7:1101:6215:4551#TGACCAAT/2
+GTTGATGCCGACCATGGCCTCCCCGAGTCCGGCGCTGACGTCGGCGAGGATGG
++
+_a_e`ccaeeegeafffhh[ffghd_cY^XZefff_d[`gGXKXW]_aaTX_^
+@FCC2HFRACXX:7:1101:9701:4567#TGACCAAT/2
+GGCAGATGGATGGATGGGTGATACAAATATATGAGAGAAGATGACGATGGTGGAGCGGATAGGGTTGTTCACTCGGATCTGGCGGCGTATCGC
++
+a_beeceegggggfghih`ffbgghhhffbffbfgfgfhdfdffghfghigegdgfhi_addeeZabbbbcbcbcaZ^ac[bbacXaTT^^a_

data/test/test_bin.rb

@@ -10,48 +10,47 @@ class TestTransrateBin < Test::Unit::TestCase
     end
 
     teardown do
-      files = ["150uncovered.l.fq.150uncovered.r.fq.assembly.2.bai",
-      "150uncovered.l.fq.150uncovered.r.fq.assembly.2.bam",
-      "150uncovered.l.fq.150uncovered.r.fq.assembly.2.sam",
-      "150uncovered.l.fq.150uncovered.r.fq.assembly.2.sorted.bam",
-      "assembly.2.1.bt2", "assembly.2.2.bt2", "assembly.2.3.bt2",
-      "assembly.2.4.bt2", "assembly.2.fa.coverage",
+      files = ["150uncovered.l.fq.150uncovered.r.fq.sorghum_transcript.bam.bai",
+      "150uncovered.l.fq.150uncovered.r.fq.sorghum_transcript.bam",
+      "sorghum_transcript_into_Os.protein.2.1.blast",
       "assembly.2_into_Os.protein.2.1.blast",
-      "assembly.2.nhr", "assembly.2.nin", "assembly.2.nsq",
-      "assembly.2.rev.1.bt2",  "assembly.2.rev.2.bt2",
-      "Os.protein.2_into_assembly.2.2.blast",
+      "sorghum_transcript.nhr", "sorghum_transcript.nin", "sorghum_transcript.nsq",
+      "Os.protein.2_into_sorghum_transcript.2.blast",
+      "assembly.2.nhr",  "assembly.2.nin",  "assembly.2.nsq",
       "Os.protein.2.phr",  "Os.protein.2.pin",  "Os.protein.2.psq",
-      "supported_bridges.csv",
-      "transrate_assemblies.csv",
-      "transrate_contigs.csv"]
+      "transrate_assemblies.csv", "params.xprs",
+      "sorghum_transcript.fa_results.xprs",
+      "sorghum_transcript.fa_bam_info.csv",
+      "transrate_sorghum_transcript.fa_contigs.csv",
+      "150uncovered.l.fq-150uncovered.r.fq-read_count.txt",
+      "150uncovered.l.fq.150uncovered.r.fq.sorghum_transcript.merged.sorted.bam"]
       files.each do |file|
         File.delete(file) if File.exist?(file)
       end
+      `rm -rf sorghum_transcript`
     end
 
     should "run help" do
       c=Transrate::Cmd.new("bundle exec bin/transrate --help")
       c.run
-      assert_equal 1751, c.stdout.length, "stdout"
+      assert c.stdout =~ /DESCRIPTION/
       assert_equal true, c.status.success?, "exit status"
     end
 
     should "fail on non existent assembly files" do
       c=Transrate::Cmd.new("bundle exec bin/transrate --assembly foo.fasta")
       c.run
-      assert_equal 163, c.stderr.length, "stderr"
       assert_equal false, c.status.success?, "exit success"
     end
 
     should "fail on non existent reference files" do
       c=Transrate::Cmd.new("bundle exec bin/transrate --reference foo.fasta")
       c.run
-      assert_equal 104, c.stderr.length, "error"
       assert_equal false, c.status.success?, "exit status"
     end
 
     should "run on test data" do
-      assembly = File.join(File.dirname(__FILE__), 'data', 'assembly.2.fa')
+      assembly = File.join(File.dirname(__FILE__), 'data', 'sorghum_transcript.fa')
       reference = File.join(File.dirname(__FILE__), 'data', 'Os.protein.2.fa')
       left = File.join(File.dirname(__FILE__), 'data', '150uncovered.l.fq')
       right = File.join(File.dirname(__FILE__), 'data', '150uncovered.r.fq')
@@ -62,8 +61,9 @@ class TestTransrateBin < Test::Unit::TestCase
       c = Transrate::Cmd.new("#{cmd}")
       c.run
       assert_equal true, c.status.success?, "exit status"
-      assert File.exist?("transrate_assemblies.csv"), "csv file doesn't exit"
-      assert File.exist?("transrate_contigs.csv"), "csv file doesn't exit"
+      assert File.exist?("transrate_assemblies.csv"), "csv file doesn't exist"
+      assert File.exist?("transrate_sorghum_transcript.fa_contigs.csv"),
+             "contig csv file doesn't exist"
       hash = {}
       CSV.foreach("transrate_assemblies.csv", :headers => true,
                                    :header_converters => :symbol,
@@ -74,9 +74,38 @@ class TestTransrateBin < Test::Unit::TestCase
           hash[header]=field
         end
       end
-      assert_equal 10331, hash[:n_bases], "number of bases"
-      assert_equal 1566, hash[:n50], "n50"
-      assert_equal 10, hash[:n_refs_with_crbb], "number of crb hits"
+      assert_equal 1555, hash[:n_bases], "number of bases"
+      assert_equal 823, hash[:n50], "n50"
+      assert_equal 0, hash[:n_refs_with_crbb], "number of crb hits"
+    end
+
+    should "run on test data with comma separated list of fastq files" do
+      assembly = File.join(File.dirname(__FILE__), 'data', 'sorghum_transcript.fa')
+      left = []
+      right = []
+      left << File.join(File.dirname(__FILE__), 'data', '150uncovered.l.fq')
+      left << File.join(File.dirname(__FILE__), 'data', 'bridging_reads.l.fastq')
+      right << File.join(File.dirname(__FILE__), 'data', '150uncovered.r.fq')
+      right << File.join(File.dirname(__FILE__), 'data', 'bridging_reads.r.fastq')
+      cmd = "bundle exec bin/transrate --assembly #{assembly}"
+      cmd << " --left #{left.join(",")}"
+      cmd << " --right #{right.join(",")}"
+      c = Transrate::Cmd.new("#{cmd}")
+      c.run
+      assert_equal true, c.status.success?, "exit status"
+      assert File.exist?("transrate_assemblies.csv")
+      assert File.exist?("transrate_sorghum_transcript.fa_contigs.csv"),
+             "contig csv file doesn't exist"
+      hash = {}
+      CSV.foreach("transrate_assemblies.csv", :headers => true,
+                                   :header_converters => :symbol,
+                                   :converters => :all) do |row|
+        row.headers.zip(row.fields).each do |header, field|
+          hash[header]=field
+        end
+      end
+      assert_equal 1555, hash[:n_bases], "number of bases"
+      assert_equal 823, hash[:n50], "n50"
     end
 
     should "fail when one of multiple assemblies is missing" do