ngs-ci 0.0.1.a

data/README.md

@@ -0,0 +1,43 @@
+[![Build Status](https://travis-ci.org/MatthewRalston/SCI.png?branch=master)](https://travis-ci.org/MatthewRalston/SCI)
+
+[![Gem Version](https://badge.fury.io/rb/SCI.png)](http://badge.fury.io/rb/SCI)
+
+[![Coverage Status](https://coveralls.io/repos/MatthewRalston/SCI/badge.png)](https://coveralls.io/r/MatthewRalston/SCI)
+
+
+
+# SCI
+
+NOTE: This is a project in progress. 
+This gem will calculate a sequencing complexity index for BAM files.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'sci'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install sci --pre
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it ( https://github.com/MatthewRalston/SCI/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request
+
+## License
+GPL v3. See LICENSE.txt for details.

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |task|
+  task.rspec_opts = ['--color']
+end
+task :default => :spec

data/TODO.md

@@ -0,0 +1,31 @@
+Library Complexity Index
+
+* Summary
+This is a Ruby gem that calculates an alternative to the "pileup" format. The calculation is an average of average overlaps among reads at a particular base in the genome. Offers both stranded and unstranded calculations.
+
+* To-do list
+1. Create gem environment
+2. Create master class with options a la transrate
+   * including FR, RF, ??, or F strand specific options
+3. Create bam processing class that
+   * increments through bases (x)
+	 * calls stranded or unstranded method
+	 * adds results to list
+   * stranded method (strand chemistry)
+	 * calls methods by strand chemistry
+	 * returns calculation class results
+   * strand chemistry methods
+     * each has specific SAM flags used to acquire reads
+     * e.g. for FR chemistry
+	     * F reads are acquired according to strand
+		 * R reads are acquired and assigned according to strand of mate
+   * unstranded method
+	 * calls samtools to acquire reads from base "x"
+	 * converts to bed and sorts
+	 * returns calculation class results
+4. Create calculation class that
+   * Increments through reads (i)
+	 * Acquires reads overlapping read "i"
+     * Calculates average overlap and adds to list
+   * Averages all overlaps and returns
+6. Prints either a Nx2 matrix or 1xN matrix to file.

data/TODO.org

@@ -0,0 +1,39 @@
+* TODO
+** Square the U/L term?? Max of 100?
+** Rarefaction curves
+*** bash + samtools + resampling (100 times?)
+*** How many divisions??
+** Triangular numbers
+*** Triangular number defined as
+*** T(n) = n(n+1)/2
+*** Maximum overlap determined by function of read length L
+*** The read with most 'even' overlaps is directly in the middle
+*** EXAMPLES:
+**** Left most read (a)
+***** T(L-1)
+**** Next left-most read (b)
+***** (L-1) + T(L-1) - 1
+**** (c)
+***** (L-2) + (L-1) + T(L-1) - 2 - 1
+*** For each of j reads (aligned in best case scenario)
+**** Sum overlaps with all other reads:
+**** f(L) = 2*T(L-1) - T(J-1) - T(L - J)
+**** f(L) = (L-1)(L-1+1) - (J-1)(J-1+1)/2 - (L-J)(L-J+1)/2
+**** f(L) = L(L-1) - J*(J-1)/2 - (L-J)(L-J+1)/2
+**** f(L) = L^2 - L + (-J*J + J)/2 - (L-J)(L-J+1)/2
+**** 2f(L) = 2*(L^2) - 2L - J^2 + J - (L-J)(L-J+1)
+**** 2f(L) = 2*(L^2) - 2L - J^2 + J - (L^2 - 2LJ + L + J^2 - J)
+**** 2f(L) = 2*(L^2) - 2L - J^2 + J - L^2 + 2LJ - L - J^2 + J
+**** 2f(L) = 2*(L^2) - L^2 - J^2 - J^2 + 2LJ - 2L - L + J + J
+**** 2f(L) = L^2 - 2*(J^2) + 2LJ - 3L + 2J
+**** f(L) = -J^2 + (L^2)/2 + LJ - 3L/2 + J
+**** 2850 (triangular number T(L-1) L=76 J=1
+**** f(76) = 2850
+* Notes
+** U*O/L vs. 200*U*O/(L^2)
+** U/L is the number of unique reads at that base, length normalized
+** When U/L is 1 (maximum saturation)
+** O = L/2
+** Although, average overlap can be greater than L/2 with less reads 
+
+* Bugs

data/bin/ngs-ci

@@ -0,0 +1,125 @@
+#!/usr/bin/env ruby
+
+require 'trollop'
+require 'bio'
+require 'NGSCI'
+require 'yell'
+
+include NGSCI
+
+# Performance settings
+RUBY_GC_HEAP_GROWTH_FACTOR=2
+RUBY_GC_MALLOC_LIMIT=40000000
+RUBY_GC_MALLOC_LIMIT_MAX=75000000
+
+
+# Show the help message without arguments
+
+ARGV[0] = "--help" if ARGV.length == 0
+
+# We want clean error messages through the logger, no ugly backtraces
+# because the user doesn't care about them, unless they specifically ask for
+# them with --loglevel debug
+module Kernel
+  alias _raise raise
+
+  def raise(*a)
+    begin
+      _raise(*a)
+    rescue NGSCIError => e
+      logger.error e.message
+      logger.debug e.backtrace unless e.backtrace.nil?
+      exit 1
+    end
+  end
+end
+
+
+opts = Trollop::options do
+  version NGSCI::VERSION
+  banner <<-EOS
+  NGSCI v#{NGSCI::VERSION}
+  by Matt Ralston <mrals@udel.edu>
+  DESCRIPTION:
+  Calculates a complexity metric for each base in the genome,
+  an alternative to pileup format.
+  The complexity metric is calculated for each base in the genome as the product of a read-length normalized average read overlap and a read-length normalized number of unique reads.
+
+  Bug reports and feature requests at:
+  http://github.com/MatthewRalston/sci
+  USAGE:
+  ngs-ci --reference REFERENCE_FASTA --bam SORTED_BAM <options>
+
+  EXAMPLES:
+  # compute sci for a set of reeads
+  ngs-ci --reference genome.fa --bam aligned_reads.bam
+
+
+  OPTIONS:
+  EOS
+  opt :reference, "Reference genome in fasta format.",
+      :type => String,
+      :required => true
+  opt :bam, "Sorted bam file.",
+      :type => String,
+      :required => true
+  opt :strand, "Strand specific option. One of [FR, RF, F].",
+      :type => String
+  opt :threads, "Number of threads to use",
+      :default => 1,
+      :type => Integer
+  opt :outfile, "Prefix filename to use for CSV output",
+      :default => "sci.csv"
+  opt :loglevel, "The amount of information to print. " +
+                 "One of [error, info, warn, debug]",
+      :default => 'info'
+end
+
+####################
+# Handle commands
+####################
+# Logging
+unless %w[error info warn debug].include? opts.loglevel
+  raise NGSCIError.new "Loglevel #{opts.loglevel} is not valid. " +
+  "It must be one of: error, info, warn, debug."
+end
+
+logger.level = Yell::Level.new opts.loglevel.to_sym
+
+# Strand specific option
+if opts.strand
+  unless %w[FR RF F].include? opts.strand
+    raise NGSCIError.new "Strand specific option #{opts.strand} is invalid." +
+      " It must be one of: [FR, RF, F]"
+  end
+end
+
+# Bam and fasta files exist
+if opts.bam && opts.reference
+  if !File.exist?(opts.bam)
+    raise NGSCIIOError.new "BAM file #{opts.bam} does not exist."
+  elsif !File.exist?(opts.reference)
+    raise NGSCIIOError.new "Fasta file #{opts.reference} does not exist."
+  end
+else
+  raise NGSCIIOError.new "A sorted BAM file and a fasta file are required."
+end
+
+####################
+# Run calculation
+####################
+logger.info "Opening BAM and reference files for calculation."
+calculator = Calculator.new(opts.bam,opts.reference,strand: opts.strand,threads: opts.threads)
+
+if opts.loglevel == "debug"
+  calculator.run(runtime: true)
+else
+  calculator.run
+end
+
+
+outfile = opts.outfile
+logger.info "Writing sequencing complexity index to #{outfile}"
+calculator.export(outfile)
+
+logger.info "Calculation complete."

data/lib/NGSCI/calculator.rb

@@ -0,0 +1,289 @@
+require 'bio'
+require 'parallel'
+require 'bio-samtools'
+require 'ruby-prof'
+
+module NGSCI
+
+  # A calculator calculates the sequencing complexity index.
+  #
+  # @!attribute [r] sci
+  class Calculator
+    attr_reader :sci, :block_size, :buffer, :chroms
+
+    # A new calculator to compute the sequencing complexity index given
+    # a loaded Bio::DB::Sam object and optional thread argument.
+    #
+    # @param bam [Bio::DB::Sam] Opened bam file with loaded reference.
+    # @param threads [Int] The number of threads used to compute NGSCI.
+    # @param strand [String] One of [FR RF F] or nil for strandedness.
+    def initialize(bam, reference, strand: nil, threads: 1)
+      @block_size = 1600
+      @results = nil
+      @reference=reference
+      @bam = Bio::DB::Sam.new(:bam=>bam,:fasta=>reference)
+      unless @bam.indexed?
+        @bam.index
+      end
+      @bam.open
+      @threads = threads
+      @chroms = reference_sequences(reference)
+      read_length
+      if strand
+        unless %w(FR RF F).include?(strand)
+          raise NGSCI::NGSCIError.new "Strand specific option #{opts.strand} is invalid." +
+      " It must be one of: [FR, RF, F]"
+        end
+        @strand = strand.downcase
+      else
+        @strand = nil
+      end
+    end
+
+    # Calculation of the sequencing complexity index
+    #
+    def run(runtime: false)
+      RubyProf.start if runtime
+      # Convert each aligned read to Read clas
+      chroms={}
+      @chroms.each do |chrom,size|
+        chroms[chrom] = @strand ? {"+"=>[],"-"=>[]} : {nil=>[]}
+        disk_accesses = (size/@block_size.to_f).ceil
+=begin
+        # N O N - P A R A L L E L
+        i=0
+        while i < disk_accesses
+
+          readblock(chrom,i).each do |key,val|
+            chroms[chrom][key] += val
+          end
+          i+=1
+        end
+=end
+
+        data = Parallel.map((0...disk_accesses).to_a,:in_processes => @threads) do |i|
+          readblock(chrom,i)
+        end
+        chroms[chrom].keys.each do |key|
+          chroms[chrom][key] = data.map{|x| x[key]}.flatten(1)
+        end
+
+      end
+      
+      # Printing runtime information for optimization
+      if runtime
+        runtime=RubyProf.stop
+        printer=RubyProf::FlatPrinter.new(runtime)
+        printer.print(STDOUT)
+      end
+      @results = chroms
+    end
+
+    # Reads a single block from the disk and calculates the NGSCI
+    #
+    # @param chrom [String] The chromosome from the bam file
+    # @param i [Integer] The number of blocks that have been read
+    # @return localNGSCI [Hash<Symbol,Array>]
+    #   * :+ (Array[Integer]) The NGSCI for the + strand
+    #   * :- (Array[Integer]) The NGSCI for the - strand
+    def readblock(chrom,i)
+      reads=[]
+      results = @strand ? {"+" => [],"-" => []}: {nil => []}
+      start = [0,(i * @block_size) - @buffer].max
+      stop = [(i + 1) * @block_size, self.chroms[chrom]].min
+      @bam.fetch(chrom,start,stop) {|read| reads << convert(read)}
+      start += @buffer unless start == 0
+      reads.compact!
+      reads.sort_by!(&:start) unless reads.empty?
+      x=0
+      bases = (start...stop).to_a
+      block = stop - start
+      while x < block
+        b = bases[x]
+        aligned = reads.select{|r| r.start <= b && r.stop - 1 >= b}.group_by &:strand
+        results.keys.each do|key|
+          results[key] << [b,*sci(aligned[key] || [])]
+        end
+        x+=1
+      end
+      return results
+    end
+
+
+    # Calculates sequencing complexity index for a single base
+    # 
+    # @param reads [Array<NGSCI::Read>] A group of reads aligned to a single base.
+    # @return sci [Float] 
+    def sci(reads)
+      numreads=reads.size
+      # Groups reads by start site
+      # selects the largest read length from the groups
+      reads = reads.group_by(&:start).map{|k,v| v.max{|x,y| (x.stop-x.start).abs <=> (y.stop-y.start).abs}}
+      o = summed_overlaps(reads)
+      uniquereads = reads.size
+      return [numreads,uniquereads,(@buffer*o.to_f/@denom).round(4),(300*uniquereads*o/(2*@denom)).round(4)]
+    end
+
+    # Calculates summed overlap between a group of reads
+    #
+    # @param reads [Array<NGSCI::Read>] Array of reads
+    # @return avg_overlap [Integer] Summed overlap between reads
+    def summed_overlaps(reads)
+      numreads = reads.size
+      sum=0
+      unless numreads == 1
+        i = 0
+        while i < numreads
+          r1 = reads[i] # for each of n reads
+          sum+=reads.
+                reject{|r| r == r1}. # select the n-1 other reads
+                map{|r| overlap(r,r1)}. # calculate their overlap to r1
+                reduce(:+)
+          i+=1
+        end
+      end
+      return sum
+    end    
+
+    # Calculation of the overlap between two reads
+    # 
+    # @param read1 [NGSCI::Read] First read to be compared
+    # @param read2 [NGSCI::Read] First read to be compared
+    # @return overlap_length [Integer] Length of overlap
+    def overlap(read1,read2)
+      if read1.start > read2.start
+        if read1.stop < read2.stop # Read 1 is inside read 2
+          read1.stop - read1.start
+        else # Normal overlap
+          read2.stop - read1.start
+        end
+      else
+        if read1.stop > read2.stop # Read 2 is inside read 1
+          read2.stop - read2.start
+        else # Normal overlap
+          read1.stop - read2.start
+        end
+      end
+    end
+
+    # Loads the read length from a bam file into the @buffer variable
+    #
+    def read_length
+      buffer=0
+      stats=@bam.index_stats.select {|k,v| k != "*" && v[:mapped_reads] > 0}
+      if stats.empty?
+        raise NGSCIIOError.new "BAM file is empty! Check samtools idxstats."
+      else
+        i=0
+        lengths=[]
+        test = @block_size
+        while i <= test 
+          @bam.view do |read|
+            lengths << read.seq.size
+            i +=1
+          end
+          if i == test && lengths.size < 100
+            test += @block_size
+          end
+        end
+        @buffer = lengths.max
+        @denom = @buffer**2 * (@buffer - 1)**2
+      end
+    end
+
+    # Converts strand specific BAM read into a sequence object format
+    # Uses the @strand instance variable to determine the strand of conversion
+    # 
+    # @param read [Bio::DB::Alignment] Read to be converted.
+    # @return read [NGSCI::Read] Converted Read object
+    def convert(read)
+      unless read.query_unmapped
+        if @strand
+          return self.send(@strand.to_sym,read)
+        else
+          return newread(read)
+        end
+      end
+      return nil
+    end
+
+
+    # Converts strand specific BAM read into a sequence object format
+    # Assumes paired-end strand-specific sequencing with "fr" chemistry
+    # 
+    # @param read [Bio::DB::Alignment] Read to be converted.
+    # @return read [NGSCI::Read] Converted Read object
+    def fr(read)
+      if read.first_in_pair
+        read.query_strand ? newread(read,strand:"+") : newread(read,strand:"-")
+      else
+        read.query_strand ? newread(read,strand:"-") : newread(read,strand:"+")
+      end
+    end
+
+
+    # Converts strand specific BAM read into a sequence object format
+    # Assumes paired-end strand-specific sequencing with "rf" chemistry
+    # 
+    # @param read [Bio::DB::Alignment] Read to be converted.
+    # @return read [NGSCI::Read] Converted Read object
+    def rf(read)
+      if read.first_in_pair
+        read.query_strand ? newread(read,strand:"-") : newread(read,strand:"+")
+      else
+        read.query_strand ? newread(read,strand:"+") : newread(read,strand:"-")
+      end      
+    end
+
+
+    # Converts strand specific BAM read into a sequence object format
+    # Assumes single-end strand-specific sequencing with "f" chemistry
+    # 
+    # @param read [Bio::DB::Alignment] Read to be converted.
+    # @return read [NGSCI::Read] Converted Read object
+    def f(read)
+      read.query_strand ? newread(read,strand:"+") : newread(read,strand:"-")
+    end
+
+    # Creates a new read with optional strand argument
+    #
+    # @param read [Bio::DB::Alignment] Aligned read to be converted
+    # @param strand [String] Strand of read
+    # @return read [NGSCI::Read] Converted Read object
+    def newread(read,strand: nil)
+      Read.new(read.pos,read.pos+read.seq.size,strand: strand)
+    end
+
+    # Acquires names and sizes of reference sequences included in the bam file
+    #
+    # @param reference [String] Path to reference fasta file.
+    # @return chromosomes [Hash<Symbol,Object>] A dictionary of chromosome sizes
+    def reference_sequences(reference)
+      chromosomes={}
+      Bio::FastaFormat.open(@reference).each_entry do |f| 
+        chromosomes[f.entry_id]=f.seq.size
+      end      
+      chromosomes.select {|chrom| @bam.index_stats.keys.include?(chrom)}
+    end
+    # Exports the results to outfile
+    #
+    # @param outfile [String] Path to outfile
+    def export(outfile)
+      if @results
+        File.open(outfile,'w') do |file|
+          file.puts("Chrom,Base,Strand,Depth,Unique_Reads,Overlap,NGS-CI")
+          @results.each do |chrom,results|
+            results.each do |strand,val|
+              val.each do |x|
+                file.puts([chrom,x[0],strand,*x[1..-1]].join(","))
+              end
+            end
+          end
+        end
+        return outfile
+      else
+        return nil
+      end
+    end
+  end # End calculator class
+end

data/lib/NGSCI/cmd.rb

@@ -0,0 +1,23 @@
+require 'open3'
+
+module NGSCI
+
+  class Cmd
+
+    attr_accessor :cmd, :stdout, :stderr, :status
+
+    def initialize cmd
+      @cmd = cmd
+    end
+
+    def run
+      @stdout, @stderr, @status = Open3.capture3 @cmd
+    end
+
+    def to_s
+      @cmd
+    end
+
+  end
+
+end

data/lib/NGSCI/read.rb

@@ -0,0 +1,31 @@
+module NGSCI
+
+  # A simple read class
+  #
+  # @!attribute [r] start
+  # @!attribute [r] stop
+  # @!attribute [r] strand
+  class Read
+    attr_reader :start, :stop, :strand
+    def initialize(start,stop,strand: nil)
+=begin DEPRECATED chromosome variable
+      unless chr.is_a?(String)
+        raise NGSCIError.new "Invalid chromosome argument:\n"
+        "chr:#{chr}\tstart:#{start}\tstop:#{stop}\tstrand:#{strand}"
+      end
+=end
+      unless start.is_a?(Integer) && stop.is_a?(Integer) && stop > start
+        raise NGSCIError.new "Invalid coordinate arguments:\n"
+        "chr:#{chr}\tstart:#{start}\tstop:#{stop}\tstrand:#{strand}"
+      end
+      if strand && !%w(+ -).include?(strand)
+        raise NGSCIError.new "Invalid strand argument:\n"
+        "chr:#{chr}\tstart:#{start}\tstop:#{stop}\tstrand:#{strand}"
+      end
+      @start=start
+      @stop=stop
+      @strand=strand
+    end
+
+  end
+end

data/lib/NGSCI/version.rb

@@ -0,0 +1,3 @@
+module NGSCI
+  VERSION = "0.0.1.a"
+end

data/lib/NGSCI.rb

@@ -0,0 +1,31 @@
+require 'yell'
+
+# NGSCI stands for Sequencing Complexity Index
+# This program calculates a sequencing complexity index for each base and/or strand in a genome.
+# This program calculates this by averaging average overlaps of reads aligned to that base.
+module NGSCI
+  # For custom error handling in the future, unimplemented
+  class NGSCIError < StandardError; end
+  class NGSCIIOError < NGSCIError; end
+  class NGSCIArgError < NGSCIError; end
+
+
+  # Create the universal logger and include it in Object
+  # making the logger object available everywhere
+  format = Yell::Formatter.new("[%5L] %d : %m", "%Y-%m-%d %H:%M:%S")
+  # http://xkcd.com/1179/
+  Yell.new(:format => format) do |l|
+    l.level = :info
+    l.name = Object
+    l.adapter STDOUT, level: [:debug, :info, :warn]
+    l.adapter STDERR, level: [:error, :fatal]
+  end
+  Object.send :include, Yell::Loggable
+
+end # NGSCI
+
+# Integrate modules
+require 'NGSCI/cmd'
+require 'NGSCI/version'
+require 'NGSCI/calculator'
+require 'NGSCI/read'

data/ngs-ci.gemspec

@@ -0,0 +1,35 @@
+
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'NGSCI/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "ngs-ci"
+  spec.version       = NGSCI::VERSION
+  spec.authors       = ["Matthew Ralston"]
+  spec.email         = ["mrals89@gmail.com"]
+  spec.summary       = %q{Next Generation Sequencing Complexity Index.}
+  spec.description   = %q{Calculated a metric that estimates read complexity at each base for RNA-seq BAM files. Alternative to pileup format.}
+  spec.homepage      = ""
+  spec.license       = "GPL v3"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'trollop','~> 2.1.2'
+  spec.add_dependency 'bio-samtools', '= 2.3.2'
+  spec.add_dependency 'parallel', '~> 1.4'
+  spec.add_dependency 'yell'
+  spec.add_dependency "ruby-prof", "~> 0.15"
+  spec.has_rdoc = 'yard'
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.1"
+  #spec.add_development_dependency "guard", "~> 2.12"
+  spec.add_development_dependency "coveralls"
+  #spec.add_development_dependency "cucumber",  "~> 1.3"
+end

data/spec/lib/NGSCI_spec.rb

@@ -0,0 +1,10 @@
+require 'spec_helper'
+
+describe NGSCI do
+
+  it 'will do something' do
+    pending('something cool')
+    raise
+  end
+
+end