lederhosen 1.6.2 → 1.7.0

data/Gemfile

@@ -1,6 +1,6 @@
 source :rubygems
 
-gem 'dna', '0.1.2'
+gem "dna", :git => 'http://github.com/audy/dna.git', :tag => 'v0.2.1'
 gem 'progressbar', '0.12.0'
 gem 'thor', '0.16.0'
 

data/lederhosen.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "lederhosen"
-  s.version = "1.6.2"
+  s.version = "1.7.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Austin G. Davis-Richardson"]
-  s.date = "2012-12-13"
+  s.date = "2012-12-19"
   s.description = "Various tools for OTU clustering"
   s.email = "harekrishna@gmail.com"
   s.executables = ["lederhosen"]
@@ -18,7 +18,6 @@ Gem::Specification.new do |s|
   ]
   s.files = [
     ".rspec",
-    ".rvmrc",
     "Gemfile",
     "LICENSE.txt",
     "Rakefile",
@@ -46,6 +45,7 @@ Gem::Specification.new do |s|
     "spec/data/ILT_L_9_B_002_3.txt.gz",
     "spec/data/example.fastq",
     "spec/data/test.uc",
+    "spec/data/trimmed_sizes.txt",
     "spec/no_tasks_spec.rb",
     "spec/spec_helper.rb",
     "spec/trimmer_spec.rb"
@@ -60,14 +60,14 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<dna>, ["= 0.1.2"])
+      s.add_runtime_dependency(%q<dna>, [">= 0"])
       s.add_runtime_dependency(%q<progressbar>, ["= 0.12.0"])
       s.add_runtime_dependency(%q<thor>, ["= 0.16.0"])
       s.add_development_dependency(%q<rdoc>, ["~> 3.12"])
       s.add_development_dependency(%q<jeweler>, ["= 1.8.4"])
       s.add_development_dependency(%q<ruby-prof>, ["= 0.11.2"])
     else
-      s.add_dependency(%q<dna>, ["= 0.1.2"])
+      s.add_dependency(%q<dna>, [">= 0"])
       s.add_dependency(%q<progressbar>, ["= 0.12.0"])
       s.add_dependency(%q<thor>, ["= 0.16.0"])
       s.add_dependency(%q<rdoc>, ["~> 3.12"])
@@ -75,7 +75,7 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<ruby-prof>, ["= 0.11.2"])
     end
   else
-    s.add_dependency(%q<dna>, ["= 0.1.2"])
+    s.add_dependency(%q<dna>, [">= 0"])
     s.add_dependency(%q<progressbar>, ["= 0.12.0"])
     s.add_dependency(%q<thor>, ["= 0.16.0"])
     s.add_dependency(%q<rdoc>, ["~> 3.12"])

data/lib/lederhosen/tasks/trim.rb

@@ -10,16 +10,16 @@ module Lederhosen
     desc "trim",
          "trim reads based on quality scores"
 
-    method_option :reads_dir, :type => :string, :required => true
-    method_option :out_dir,   :type => :string, :required => true
-    method_option :pretrim,   :type => :numeric, :default => 11
-    method_option :read_type, :type => :string, :default => 'qseq'
+    method_option :reads_dir,  :type => :string, :required => true
+    method_option :out_dir,    :type => :string, :required => true
+    method_option :left_trim,  :type => :numeric, :default => 0
+    method_option :read_type,  :type => :string, :default => 'qseq'
     method_option :min_length, :type => :numeric, :default => 75
 
     def trim
       raw_reads  = options[:reads_dir]
       out_dir    = options[:out_dir]
-      pretrim    = options[:pretrim]
+      left_trim  = options[:left_trim]
       read_type  = options[:read_type]
       min_length = options[:min_length]
 
@@ -48,7 +48,7 @@ module Lederhosen
         out = File.join(out_dir, "#{File.basename(prefix)}.fasta")
 
         # create the trimmed sequence generator
-        trim_args = { :pretrim => pretrim, :min_length => min_length }
+        trim_args = { :left_trim => left_trim, :min_length => min_length }
 
         trimmer =
           if read_type == 'qseq'

data/lib/lederhosen/trimmer.rb

@@ -1,56 +1,39 @@
 module Lederhosen
 module Trimmer
 
-# Base class for trimming paired-end reads
-class PairedTrimmer < Enumerator
-  def initialize(paired_iterator, args = {})
-    @paired_iterator = paired_iterator
-    @pretrim         = args[:pretrim]
-    @min_length      = args[:min_length] || 70
-    @min             = args[:min] || 20
-    @offset          = args[:cutoff] || 64 # XXX should both be called 'cutoff'
-    @pretrim         = args[:pretrim] || false
-  end
+##
+# Code used for sequence trimming
+#
+# - PairedTrimmer
+# - HuangTrimmer
+# - ProbabilityTrimmer
+# - QSEQTrimmer
+#
+# Some major refactoring needs to get done here
+#
 
-  def each(&block)
-    @paired_iterator.each_with_index do |a, i|
-      seqa = trim_seq a[0], :pretrim => @pretrim
-      seqb = trim_seq a[1], :pretrim => @pretrim
-      unless [seqa, seqb].include? nil
-        if seqb.length >= @min_length && seqa.length >= @min_length
-          seqb = reverse_complement(seqb) # experiment-specific?
-          a = Fasta.new :name => "#{i}:0", :sequence => seqa
-          b = Fasta.new :name => "#{i}:1", :sequence => seqb
-          block.yield a
-          block.yield b
-        else # we just skip bad reads entirely
-          next
-        end
-      else
-        next
-      end
-    end
-  end
+# HaungTrimmer
+#
+# class that has the trim function. Used in mixins
+# this trim function is based on the function documented
+# in the paper:
+#   Huang X, Wang J, Aluru S, Yang SP, Hillier L. (2003). PCAP:
+#   a whole-genome assembly program. Genome Res 13:
+#   2164–2170.
+#
+# The implementation is a direct copy from the perl implementation
+# implemented in Pangea 1.0:
+#   PANGEA: pipeline for analysis of next generation amplicons
+#   A Giongo, DB Crabb, AG Davis-Richardson - ISME , 2010
+#
+class HuangTrimmer
 
-  # reverse complement a DNA sequence
-  # assumes only GATCN nucleotides
-  def reverse_complement(s)
-    s.reverse.tr('GATCNgatcn','CTAGNctagn')
+  def initialize(args={})
+    @min = args[:min]
+    @offset = args[:offset]
   end
 
-  # this method does the actual trimming. It is a class method
-  # so you can use it if you don't want to initialize a PairedTrimmer
-  def trim_seq(dna, args={})
-
-    # trim primers off of sequence
-    # XXX this is experiment-specific and needs to be made
-    # into a parameter
-    if @pretrim
-      dna.sequence = dna.sequence[@pretrim..-1]
-      dna.quality  = dna.quality[@pretrim..-1]
-    end
-
-    dna.sequence.gsub! '.', 'N'
+  def trim_seq(dna)
 
     _sum, _max, first, last, start, _end = 0, 0, 0, 0, 0
 
@@ -66,14 +49,130 @@ class PairedTrimmer < Enumerator
       end
     end
 
-    dna.sequence[start, _end - start] rescue nil
+    begin
+      dna.sequence[start, _end - start].gsub('.', 'N')
+    rescue
+      nil
+    end
   end
+end
 
+#
+# return the longest string starting from the left side
+# where the PROBABILITY OF ERROR as computed from the PHRED
+# scores does not go above a certain cutoff
+# (default is 0.005)
+#
+class ProbabilityTrimmer
+
+  def initialize(args = {})
+    @cutoff = args[:cutoff] || 0.005
+    @min = args[:min]
+    @seqtech = args[:seq_tech] || fail
+    # must be illumina, sanger or solexa
+  end
+
+  def trim_seq(dna)
+    trim_coord = dna.sequence.size
+    probabilities = dna.send(:"#{@seqtech}_probabilities")
+    probabilities.each_with_index do |q, i|
+      if q > @cutoff
+        trim_coord = i
+        break
+      end
+    end
+
+    begin
+      dna.sequence[0..trim_coord].gsub('.', 'N')
+    rescue
+      nil
+    end
+  end
+end
+
+#
+# Base class for trimming paired-end reads
+#
+class PairedTrimmer < Enumerator
+
+  def initialize(args = {})
+    @pretrim    = args[:pretrim]
+    # TODO
+    # need to be able to trim from left, right of pairs
+    # thinking about specifying a "trimming language"
+    #
+    # Something like:
+    #
+    # --trim="5L0 0L3"
+    # --trim="0L4 2L6"
+    #
+    # also thinking about breaking all of this trimming stuff
+    # out into its own package. (to be more unixy and stuff ;)
+    #
+    @min_length = args[:min_length] || 70
+    @min         = args[:min] || 20
+    @offset      = args[:cutoff] || 64 # XXX should both be called 'cutoff'
+    @left_trim   = args[:left_trim] || 0 # trim adapter sequence
+    @skip_ambig  = args[:skip_ambiguous] || false
+    @trimmer     = args[:trimmer] || ProbabilityTrimmer.new(:min => @min,
+                                                           :offset => @offset,
+                                                           :seq_tech =>
+                                                           :illumina)
+  end
+
+  def each(&block)
+
+    skipped_because_singleton = 0
+    skipped_because_length = 0
+    skipped_because_ambig = 0
+
+    @paired_iterator.each_with_index do |a, i|
+      seqa = @trimmer.trim_seq(a[0])[@left_trim..-1] rescue nil # trim adapter sequence
+      seqb = @trimmer.trim_seq a[1]
+
+      # make sure sequences are good
+      # (both pairs survived and both are at least min_length long)
+      # optionally skip reads that contain ambiguous nucleotides (N)
+      if [seqa, seqb].include? nil
+        skipped_because_singleton += 1
+      elsif !(seqb.length >= @min_length && seqa.length >= @min_length)
+        skipped_because_length += 1
+      elsif @skip_ambig and (seqb =~ /N/ or seqa =~ /N/)
+        skipped_because_ambig
+      else # reads are good
+        #
+        # TODO
+        # this is experiment specific. I save memory down the road
+        # by having both of the reads in the forward orientation
+        # but depending on the sequencing technology/pipeline
+        # this may change.
+        #
+        # I'm planning on removing the trimming steps from lederhosen
+        # for their own gem. With that, this will go too.
+        #
+        seqb = reverse_complement(seqb)
+        
+        # Create and yield new fasta objects
+        # Perhaps this is slow?
+        a = Fasta.new :name => "#{i}:0", :sequence => seqa
+        b = Fasta.new :name => "#{i}:1", :sequence => seqb
+        block.yield a
+        block.yield b
+      end
+    end
+  end
+
+  # reverse complement a DNA sequence
+  # assumes only GATCN nucleotides
+  def reverse_complement(s)
+    s.reverse.tr('GATCNgatcn','CTAGNctagn')
+  end
 end
 
 #
 # Yields trimmed fasta records given an input
 # interleaved, paired-end fastq file
+#
 class InterleavedTrimmer < PairedTrimmer
 
   def initialize(interleaved_file, args = {})
@@ -88,15 +187,16 @@ class InterleavedTrimmer < PairedTrimmer
       end
 
     reads = Dna.new handle
-    iterator = reads.each_slice(2)
-
-    super(iterator, args)
+    @paired_iterator = reads.each_slice(2)
 
+    super(args)
   end
 end
 
+#
 # Yield trimmed fasta records given an two separate
 # paired QSEQ files
+#
 class QSEQTrimmer < PairedTrimmer
   def initialize(left_file, right_file, args = {})
     # create an iterator that yields paired records
@@ -112,9 +212,9 @@ class QSEQTrimmer < PairedTrimmer
     left_file_reads  = Dna.new left_handle
     right_reads = Dna.new right_handle
 
-    iterator = left_file_reads.zip(right_reads)
+    @paired_iterator = left_file_reads.zip(right_reads)
 
-    super(iterator, args)
+    super(args)
 
     left_handle.close
     right_handle.close

data/lib/lederhosen/version.rb

@@ -1,9 +1,9 @@
 module Lederhosen
   module Version
     MAJOR = 1
-    MINOR = 6
-    CODENAME = 'Sauerkraut' # changes for minor versions
-    PATCH = 2
+    MINOR = 7
+    CODENAME = 'Franziskaner' # changes for minor versions
+    PATCH = 0
 
     STRING = [MAJOR, MINOR, PATCH].join('.')
   end

data/readme.md

@@ -2,11 +2,18 @@
 
 # Lederhosen
 
-OTU clustering for rRNA amplicons. Lederhosen is intended to be simple, robust and easy to use.
+Lederhosen is a set of tools for OTU clustering rRNA amplicons using Robert Edgar's USEARCH.
 
-### Why not QIIME?
+It handles quality control of raw sequence data, running USEARCH, and creating and filtering tables.
 
-QIIME is great but imagine for a moment, if you will, a world where there was only one web browser.
+Lederhosen is not a pipeline but rather a set of tools broken up into tasks. Tasks are invoked by running `lederhosen TASK ...`.
+
+Lederhosen is designed with the following "pipeline" in mind:
+
+1. Quality control of sequence data.
+2. Clustering sequences to centroid or reference sequences (read: database)
+3. Generating tables from USEARCH output.
+4. Filtering tables to remove small or insignificant OTUs.
 
 ### About
 
@@ -20,13 +27,18 @@ using paired and non-paired end short reads such as those produced by Illumina (
 ### Features
 
 - Sequence trimming (paired-end Illumina).
-- Parallel, referenced-based clustering to TaxCollector using USEARCH
+- Parallel, referenced-based clustering to TaxCollector using USEARCH.
+- Queue-agnostic support for running jobs on clusters.
+- Support for RDP, TaxCollector or GreenGenes databases.
 - Generation and filtering of OTU abundancy matrices.
 
 ### Installation
 
 0. Obtain & Install [USEARCH](http://www.drive5.com/) (32bit is fine for non-commercial use)
-2. Get a copy of [TaxCollector](http://github.com/audy/taxcollector) or [GreenGenes](http://greengenes.lbl.gov) 16S database
+2. Get a database:
+  - [TaxCollector](http://github.com/audy/taxcollector)
+  - [GreenGenes](http://greengenes.lbl.gov) 16S database
+  - File an [issue report](https://github.com/audy/lederhosen/issues) or pull request ;) to request support for a different database.
 3. Install Lederhosen by typing:
 
     `sudo gem install lederhosen`
@@ -48,7 +60,9 @@ You can also trim interleaved, paired-end FASTQ files:
 
     lederhosen trim --reads_dir=reads/*.fastq --out_dir=trimmed/ read-type='fastq'
 
-(WARNING: by default, Lederhosen trims off the first 11 nucleotides. This was to remain backwards-compatibility with an experiment-specific version. To disable use `--pretrim 0`)
+Lederhosen will also trim off adapter sequences from the 5' end of the "left" read with the `--left-trim` option.
+
+    lederhosen trim --reads_dir=reads/*.fastq --out_dir=trimed/ --read-type='fastq' --left-trim=11
 
 ### Create Database
 
@@ -94,8 +108,28 @@ This will create the files:
 You can get the representative sequences for each cluster using the `get_reps` tasks. This will extract the representative sequence from
 the __database__ you ran usearch with. Make sure you use the same database that you used when running usearch.
 
-    lederhosen get_reps --input=clusters.uc --database=taxcollector.fa --output=representatives.fasta
+```bash
+lederhosen get_reps \
+  --input=clusters.uc \
+  --database=taxcollector.fa \
+  --output=representatives.fasta
+```
 
 You can get the representatives from more than one cluster file using a glob:
 
-    lederhosen get_reps --input=*.uc --database=taxcollector.fa --output=representatives.fasta
+```bash
+lederhosen get_reps \
+  --input=*.uc \
+  --database=taxcollector.fa \
+  --output=representatives.fasta
+```
+
+## Acknowledgements
+
+- Lexi, Vinnie and Kevin for beta-testing and putting up with bugs
+- The QIIME project for inspiration
+- Sinbad Richardson for the Lederhosen Guy artwork
+
+## Please Cite
+
+Please cite this GitHub repo (https://github.com/audy/lederhosen) with the version you used (type `lederhosen version`) unless I publish a paper. Then cite that.

data/spec/data/trimmed_sizes.txt

@@ -0,0 +1,100 @@
+99
+91
+100
+85
+100
+91
+100
+81
+100
+81
+100
+91
+100
+91
+100
+91
+100
+91
+100
+91
+100
+91
+100
+91
+92
+91
+100
+91
+99
+91
+100
+91
+100
+81
+100
+81
+100
+91
+100
+91
+100
+91
+99
+91
+100
+91
+100
+91
+100
+81
+100
+91
+98
+91
+40
+91
+96
+91
+35
+81
+100
+91
+100
+91
+100
+91
+100
+91
+100
+91
+100
+91
+100
+91
+100
+91
+49
+91
+100
+91
+100
+91
+100
+91
+100
+91
+89
+81
+100
+91
+100
+91
+100
+91
+100
+91
+100
+91
+100
+91

data/spec/trimmer_spec.rb

@@ -2,13 +2,49 @@ require 'spec_helper'
 
 describe Lederhosen::Trimmer do
 
-  describe Lederhosen::Trimmer::PairedTrimmer do
+  describe Lederhosen::Trimmer::PairedTrimmer
+
+  describe Lederhosen::Trimmer::ProbabilityTrimmer do
+    let :sequence_trimmer do
+      # default cutoff should be 0.005
+      Lederhosen::Trimmer::ProbabilityTrimmer.new :seq_tech => :illumina
+    end
+
+    it 'can be created' do
+      sequence_trimmer.should_not be_nil
+    end
+
+    it 'trims records as expected'
+
+  end
+
+  describe Lederhosen::Trimmer::HuangTrimmer do
+
+    let :sequence_trimmer do
+      Lederhosen::Trimmer::HuangTrimmer.new(:offset => 64, :min => 20)
+    end
+
+    it 'trims records as expected' do
+
+      trimmed_sizes = File.readlines('spec/data/trimmed_sizes.txt').map &:to_i
+
+      File.open('spec/data/example.fastq') do |handle|
+        records = Dna.new handle
+        records.each do |record|
+          trimmed_record = sequence_trimmer.trim_seq record
+          trimmed_record.size.should == trimmed_sizes.shift
+        end
+      end
+    end
 
   end
 
   describe Lederhosen::Trimmer::QSEQTrimmer do
 
-    let(:qseq_trimmer) { Lederhosen::Trimmer::QSEQTrimmer.new 'spec/data/ILT_L_9_B_001_1.txt.gz', 'spec/data/ILT_L_9_B_001_3.txt.gz' }
+    let :qseq_trimmer do
+      Lederhosen::Trimmer::QSEQTrimmer.new 'spec/data/ILT_L_9_B_001_1.txt.gz',
+                                           'spec/data/ILT_L_9_B_001_3.txt.gz'
+    end
 
     it 'can be initialized' do
       qseq_trimmer.should_not be_nil
@@ -27,7 +63,9 @@ describe Lederhosen::Trimmer do
 
   describe Lederhosen::Trimmer::InterleavedTrimmer do
 
-    let(:interleaved_trimmer) { Lederhosen::Trimmer::InterleavedTrimmer.new 'spec/data/example.fastq' }
+    let :interleaved_trimmer do
+      Lederhosen::Trimmer::InterleavedTrimmer.new 'spec/data/example.fastq'
+    end
 
     it 'can be initialized' do
       interleaved_trimmer.should_not be_nil

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lederhosen
 version: !ruby/object:Gem::Version
-  version: 1.6.2
+  version: 1.7.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,24 +9,24 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-13 00:00:00.000000000 Z
+date: 2012-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dna
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - '='
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - '='
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.1.2
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: progressbar
   requirement: !ruby/object:Gem::Requirement
@@ -116,7 +116,6 @@ extra_rdoc_files:
 - LICENSE.txt
 files:
 - .rspec
-- .rvmrc
 - Gemfile
 - LICENSE.txt
 - Rakefile
@@ -144,6 +143,7 @@ files:
 - spec/data/ILT_L_9_B_002_3.txt.gz
 - spec/data/example.fastq
 - spec/data/test.uc
+- spec/data/trimmed_sizes.txt
 - spec/no_tasks_spec.rb
 - spec/spec_helper.rb
 - spec/trimmer_spec.rb
@@ -162,7 +162,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -3699960078851600472
+      hash: -1116066410733680786
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:

data/.rvmrc

@@ -1 +0,0 @@
-rvm use 1.8.7@lederhosen --create