basil 0.0.1 → 0.0.2

data/.gitignore

@@ -0,0 +1,2 @@
+Gemfile.lock
+pkg/

data/.rspec

@@ -0,0 +1 @@
+rspec -c -f d

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.8.7@basil --create

data/Gemfile

@@ -0,0 +1,7 @@
+source :rubygems
+gemspec
+gem "dna"
+gem "awesome_print"
+gem "trollop"
+gem "rspec"
+gem "awesome_print"

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/basil.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require 'basil/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'basil'
+  s.version     = Basil::VERSION
+  s.authors     = ["Austin G. Davis-Richardson"]
+  s.email       = ["harekrishna@gmail.com"]
+  s.homepage    = "http://github.com/audy/basil"
+  s.summary     = 'HTPS demultiplexer'
+  s.description = 'Split multiplexed high-throughput sequencing reads by barcode.'
+  s.rubyforge_project = "basil"
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_dependency('dna')
+  s.add_dependency('trollop')
+  s.add_dependency('rspec')
+  s.add_dependency('bundler')
+end

data/bin/basil

@@ -1,29 +1,5 @@
 #!/usr/bin/env ruby
 
-require 'ostruct'
-require 'optparse'
-require 'pp'
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'basil'))
 
-require 'bundler'
-Bundler.require :default
-
-opts = Trollop::options do
-
-  version "Basil PRE-RELEASE"
-  banner <<-EOS
-Basil is an HTPS demultiplexer
-
-Usage:
-    basil [options] -i <reads/directory> -o <output directory>
-where [options] are:
-  EOS
-
-  opt :barcodes, 'barcodes definition file', short: 'b',  type: String
-  opt :reads, 'reads (fastq/fasta/qseq)', short: 'i'
-  opt :illumina, 'Illumina output directory', short: 'd'
-  opt :pretrim, 'trim N nucleotides from start', type: Integer
-  opt :afttrim, 'trim N nucleotides from end', type: Integer
-  opt :out, 'output directory', short: 'o', type: String
-end
-
-filename = ARGV.shift
+Basil::CLI.run!

data/lib/basil.rb

@@ -1,3 +1,10 @@
+require 'pp'
+require 'awesome_print'
+require 'bundler'
+
+Bundler.require :default
+
+
 Dir.glob(File.join(File.dirname(__FILE__), 'basil', '*.rb')).each do |f|
   require f
 end

data/lib/basil/barcodes.rb

@@ -0,0 +1,3 @@
+class Barcodes
+  # TODO add commonly used barcodes where copyright permits.
+end

data/lib/basil/basil.rb

@@ -0,0 +1,60 @@
+module Basil
+
+  class Basil
+
+    #
+    # Gets Basil ready, must specify a barcodes hash with Basil.new barcodes
+    #
+    # Barcodes hash has regexp for values. If not, strings are converted to regexp
+    # Barcode names are keys, used to generate filenames.
+    #
+    def initialize(barcodes)
+      @barcodes = Hash.new
+      barcodes.each_pair do |k, v|
+        @barcodes[k] = Regexp.new "^#{v}", true
+      end
+      @barcodes
+    end
+
+    #
+    # Finds the barcode (if present)
+    #
+    # If the barcode is present, returns 'name',
+    # and the sequence with the barcode removed
+    # If not, returns nil
+    #
+    def recognize(string)
+      matches = @barcodes.each_pair.collect do |k, v|
+        k if string[v]
+      end
+
+      matches.compact!
+
+      if matches.length > 1
+        raise Exception, "sequence #{string} has more than one match"
+      elsif matches.length == 0
+        nil
+      else
+        barcode = matches.first
+        sequence = string.gsub(@barcodes[barcode], '')
+        [barcode, sequence]
+      end
+    end
+
+    #
+    # Parses a barcodes file
+    #
+    # barcodes are specified in CSV file
+    # (unless specified otherwise with :sep => '')
+    # barcode_name,sequence
+    # the barcode_name will be used to generate the filename
+    #
+    def self.parse_barcodes(handle, args={})
+      sep = args[:sep] || ","
+      barcodes = handle.each.collect do |line|
+        name, barcode = line.strip.split(sep)
+      end
+      Hash[*barcodes.flatten]
+    end
+  end
+end

data/lib/basil/buffer.rb

@@ -0,0 +1,50 @@
+module Basil
+  class Buffer
+
+    #
+    # Create a new buffer
+    #
+    def initialize(args={})
+      @buffer = Hash.new { |h, k| h[k] = Array.new }
+      @buffer_max = args[:buffer_max] || 100_000
+    end
+
+    #
+    # Add an object to the buffer
+    #
+    def add_to(bucket, obj)
+
+      @buffer[bucket] << obj
+
+      if @buffer[bucket].length > @buffer_max
+        # write out
+        File.open(bucket, 'a+') do |out|
+          @buffer[bucket].each do |v|
+            out.puts v
+          end
+        end
+
+        # clear that bucket
+        @buffer[bucket].clear
+      end
+    end
+
+    def [](k)
+      @buffer[k]
+    end
+
+    #
+    # Writes out leftover objects
+    #
+    def finalize
+      @buffer.each_key do |bucket|
+        File.open(bucket, 'a+') do |out|
+          @buffer[bucket].each do |v|
+            out.puts v
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/basil/cli.rb

@@ -0,0 +1,99 @@
+module Basil
+  class CLI
+    class << self
+
+    def run!
+
+      opts           = parse_arguments
+      barcodes       = parse_barcodes opts[:barcodes]
+      out_dir        = opts[:out]
+      buffer         = Buffer.new
+      basil          = Basil.new(barcodes)
+      handles        = Array.new
+
+      if opts[:illumina]
+        if pretrim
+          raise "pretrim is not supported with paired-end data"
+        end
+      else
+        reads_handle   = File.open(opts[:reads])
+        handles       << reads_handle
+        records_format = records.format
+        pretrim        = opts[:pretrim] || 0
+      end
+
+      if File.exist? out_dir
+        $stderr.puts "#{out_dir} already exists -- delete or move!"
+        exit
+      else
+        Dir.mkdir(out_dir)
+      end
+
+      records.each do |record|
+        
+        match = basil.recognize record.sequence
+        barcode, trimmed_sequence = match if match
+
+        unless match
+          barcode = 'unknown'
+          trimmed_sequence = record.sequence
+        end
+
+        new_record = Fasta.new :name => record.name, :sequence => trimmed_sequence
+        buffer.add_to File.join(out_dir, barcode + ".#{records_format}"), new_record
+      end
+
+      # Finish up
+      buffer.finalize
+      handles.collect { |x| x.close }
+    end
+
+    def parse_barcodes barcodes
+      File.open(barcodes) { |h| Basil.parse_barcodes h }
+    end
+
+    #
+    # For parsing illumina-generated output directories containing
+    # paired QSEQ files
+    #
+    def parse_illumina_directory dir
+      files = Dir[File.join(dir, '*')]
+
+      files.collect do |x|
+        m = File.basename(x).match(/s_(\d)_(\d)_(\d*)_qseq\.txt/)
+        { :lane => m[1], :pair => m[2], :n => m[3], :filename => File.join(dir, m[0]) }
+      end.group_by { |x| x[:n] }
+    end
+
+    def parse_arguments
+      opts = Trollop::options do
+
+        version "PRE-RELEASE"
+        banner <<-EOS
+Basil is an HTPS demultiplexer
+
+Version: #{version}
+
+Usage:
+    basil [options] -i <reads/directory> -o <output directory>
+where [options] are:
+        EOS
+
+        opt :barcodes, 'barcodes definition file', :short => 'b', :type => String
+        opt :reads, 'reads (fastq/fasta/qseq)', :short => 'i', :type => String
+        opt :illumina, 'Illumina output directory', :short => 'd'
+        opt :out_fmt, 'Output format (default: same format as input)', :short => 'f'
+        opt :pretrim, 'trim N nucleotides from start before searching for barcode', :type => Integer
+        opt :out, 'output directory', :short => 'o', :type => String
+      end
+
+      filename = ARGV.shift
+      Trollop::die :barcodes, "must specify barcodes" if opts[:barcodes].nil?
+      Trollop::die :reads, "must specify reads or Illumina directory" if !(opts[:reads] || opts[:illumina])
+      Trollop::die "cannot specify both reads and Illumina directory" if opts[:reads] && opts[:illumina]
+      Trollop::die :out, "must specify output directory" if !opts[:out]
+      opts
+    end
+  end 
+  end
+end

data/lib/basil/version.rb

@@ -0,0 +1,3 @@
+module Basil
+  VERSION = "0.0.2"
+end

data/readme.md

@@ -6,6 +6,42 @@ Yet another tool for splitting barcoded sequences. It had to be done.
 
     gem install basil
 
+## Features
+
+- Trims and separates by barcode
+- Supports unlimited number of barcodes
+- Trimming from left/right end of read
+- Barcodes provided as simple .csv file
+- Support input filetypes:
+    - Fastq
+    - Fasta
+
+## TODO
+
+- Support regexp in barcodes input.
+- QSEQ (Illumina)
+
 ## Usage
 
-    basil -h
+    Usage:
+        basil [options] -i <reads/directory> -o <output directory>
+    where [options] are:
+      --barcodes, -b <s>:   barcodes definition file
+             --reads, -i:   reads (fastq/fasta/qseq)
+          --illumina, -d:   Illumina output directory
+       --pretrim, -p <i>:   trim N nucleotides from start
+       --afttrim, -a <i>:   trim N nucleotides from end
+           --out, -o <s>:   output directory
+           --version, -v:   Print version and exit
+              --help, -h:   Show this message
+
+## Barcodes file
+
+Barcodes are specified in a simple CSV file:
+
+    barcode_name,barcode_sequence
+    my_awesome_barcode,gggggggg
+    my_awesomer_barcode,aaaaaaaa
+    my_awesomest_barcode,cccccccc
+
+(Eventually, I will add built-in support for commonly-used barcodes)

data/spec/basil_spec.rb

@@ -1,6 +1,6 @@
 require 'spec_helper'
 
-describe Basil do
+describe Basil::Basil do
 
   let (:barcodes) {
     {
@@ -11,28 +11,47 @@ describe Basil do
     }
   }
 
-  let (:basil) { Basil.new barcodes }
+  let (:basil) { Basil::Basil.new barcodes }
 
-  let (:dna) { 'ggggtcatcatcatcatcatcatca' }
+  let (:dna) { 'ggggnnnn' }
 
   it 'can be created' do
     barcodes.should_not raise_error TypeError
   end
 
   it 'raises an error if duplicate barcodes are specified' do
-    Basil.new({ 1 => 'gggg', 2 => 'gggg' }).should raise_error
+    Basil::Basil.new({ 1 => 'gggg', 2 => 'gggg' }).should raise_error
   end
 
   it '#recognize returns the name of matched barcode' do
-    basil.recognize(dna).should == 1
+    basil.recognize(dna).first.should == 1
+  end
+
+  it 'accepts regexp for barcodes' do
+    basil = Basil::Basil.new({ 1 => /^.GATC/i})
+    basil.recognize("NGATC").first.should == 1
+  end
+
+  it 'preserves case sensitivity of regexp' do
+    basil = Basil::Basil.new(1 => /^.GATC/)
+    basil.recognize("Ngatc").should be_nil
   end
 
   it '#recognize is case insensitive' do
-    basil.recognize(dna.upcase!).should == 1
+    basil.recognize(dna.upcase!).first.should == 1
   end
 
-  it '#recognize does not recognize barcodes in the middle of sequences' do
+  it '#recognize ^ is prepended to barcode regexp if strings are provided' do
     basil.recognize('aggggaaa').should be_nil
   end
-  
-end
+
+  it 'trims barcode from sequence' do
+    basil.recognize(dna).last.should == 'nnnn'
+  end
+
+  it 'can parse a barcodes.csv file producing a hash' do
+    parsed_barcodes = File.open('spec/data/barcodes.csv') { |h| Basil::Basil.parse_barcodes h }
+    parsed_barcodes.should == { 'a' => 'a'*7, 'g' => 'g'*7, 'c' => 'c'*7, 't' => 't'*7 }
+  end
+
+end

data/spec/buffer_spec.rb

@@ -1,8 +1,8 @@
 require 'spec_helper'
 
-describe Buffer do
+describe Basil::Buffer do
 
-  let (:buffer) { Buffer.new }
+  let (:buffer) { Basil::Buffer.new }
 
   it 'can be created' do
     buffer.should_not raise_error TypeError

data/spec/cli_spec.rb

@@ -0,0 +1,28 @@
+describe Basil::CLI do
+
+  it 'should be executable' do
+    `bin/basil 2> /dev/null`
+    $?.should_not == 0
+  end
+
+  it 'should have a help menu' do
+    `bin/basil --help`.include?('help').should be_true
+  end
+
+  it 'should parse barcodes.csv files' do
+    barcodes = Basil::CLI.parse_barcodes 'spec/data/barcodes.csv'
+    barcodes.should == { "a" => "aaaaaaa", "g" => "ggggggg", "c" => "ccccccc", "t" => "ttttttt"}
+  end
+
+  it 'can parse an illumina output directory' do
+    files = Basil::CLI.parse_illumina_directory 'spec/data/illumina'
+    directory = {
+      "001" => [
+        {:lane=>"1", :pair=>"1", :n=>"001", :filename=>"spec/data/illumina/s_1_1_001_qseq.txt"},
+        {:lane=>"1", :pair=>"2", :n=>"001", :filename=>"spec/data/illumina/s_1_2_001_qseq.txt"},
+        {:lane=>"1", :pair=>"3", :n=>"001", :filename=>"spec/data/illumina/s_1_3_001_qseq.txt"}
+      ]
+    }
+    files.should == directory
+  end
+end

data/spec/data/barcodes.csv

@@ -0,0 +1,4 @@
+a,aaaaaaa
+g,ggggggg
+c,ccccccc
+t,ttttttt

data/spec/data/illumina/s_1_1_001_qseq.txt

@@ -0,0 +1,5 @@
+HWUSI-EAS1700R  5       4       1       975     930     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1051    933     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1233    932     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1266    945     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1373    940     0       2       ....... BBBBBBB 0

data/spec/data/illumina/s_1_2_001_qseq.txt

@@ -0,0 +1,5 @@
+HWUSI-EAS1700R  5       4       1       975     930     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1051    933     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1233    932     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1266    945     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1373    940     0       2       ....... BBBBBBB 0

data/spec/data/illumina/s_1_3_001_qseq.txt

@@ -0,0 +1,5 @@
+HWUSI-EAS1700R  5       4       1       975     930     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1051    933     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1233    932     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1266    945     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1373    940     0       2       ....... BBBBBBB 0

data/spec/data/test.fasta

@@ -0,0 +1,20 @@
+>1
+AAAAAA
+>2
+GGGGGG
+>3
+TTTTTT
+>4
+CCCCCC
+>5
+AAATTT
+>6
+TTTAAA
+>7
+GGGCCC
+>8
+CCCGGG
+>9
+GAAAAA
+>10
+CAAAAA

data/spec/data/test.fastq

@@ -0,0 +1,20 @@
+@a
+aaaaaaa
++a
+BBBBBBB
+@b
+ttttttt
++b
+BBBBBBB
+@c
+ccccccc
+@c
+BBBBBBB
+@d
+ggggggg
+@d
+BBBBBBB
+@e
+gagagag
+@e
+BBBBBBB

data/spec/data/test.qseq

@@ -0,0 +1,5 @@
+HWUSI-EAS1700R  5       4       1       975     930     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1051    933     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1233    932     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1266    945     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1373    940     0       2       ....... BBBBBBB 0

metadata

@@ -1,84 +1,155 @@
---- !ruby/object:Gem::Specification
+--- !ruby/object:Gem::Specification 
 name: basil
-version: !ruby/object:Gem::Version
-  version: 0.0.1
+version: !ruby/object:Gem::Version 
+  hash: 27
   prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
 platform: ruby
-authors:
+authors: 
 - Austin G. Davis-Richardson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2010-02-10 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
+
+date: 2012-05-14 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
   name: dna
-  requirement: &70286137539700 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: 0.0.0
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :runtime
-  prerelease: false
-  version_requirements: *70286137539700
-- !ruby/object:Gem::Dependency
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
   name: trollop
-  requirement: &70286137539140 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :runtime
-  prerelease: false
-  version_requirements: *70286137539140
-- !ruby/object:Gem::Dependency
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
   name: rspec
-  requirement: &70286137538460 !ruby/object:Gem::Requirement
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
   type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: bundler
   prerelease: false
-  version_requirements: *70286137538460
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
 description: Split multiplexed high-throughput sequencing reads by barcode.
-email: harekrishna@gmail.com
-executables:
+email: 
+- harekrishna@gmail.com
+executables: 
 - basil
 extensions: []
+
 extra_rdoc_files: []
-files:
+
+files: 
+- .gitignore
+- .rspec
+- .rvmrc
+- Gemfile
+- Rakefile
+- basil.gemspec
+- bin/basil
 - lib/basil.rb
+- lib/basil/barcodes.rb
+- lib/basil/basil.rb
+- lib/basil/buffer.rb
+- lib/basil/cli.rb
+- lib/basil/version.rb
+- readme.md
 - spec/basil_spec.rb
 - spec/buffer_spec.rb
+- spec/cli_spec.rb
+- spec/data/barcodes.csv
+- spec/data/illumina/s_1_1_001_qseq.txt
+- spec/data/illumina/s_1_2_001_qseq.txt
+- spec/data/illumina/s_1_3_001_qseq.txt
+- spec/data/test.fasta
+- spec/data/test.fastq
+- spec/data/test.qseq
 - spec/spec_helper.rb
-- readme.md
-- bin/basil
 homepage: http://github.com/audy/basil
 licenses: []
+
 post_install_message: 
 rdoc_options: []
-require_paths:
+
+require_paths: 
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement
+required_ruby_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
-required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
-  requirements:
-  - - ! '>='
-    - !ruby/object:Gem::Version
-      version: '0'
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
 requirements: []
-rubyforge_project: 
-rubygems_version: 1.8.15
+
+rubyforge_project: basil
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: HTPS demultiplexer
-test_files: []
+test_files: 
+- spec/basil_spec.rb
+- spec/buffer_spec.rb
+- spec/cli_spec.rb
+- spec/data/barcodes.csv
+- spec/data/illumina/s_1_1_001_qseq.txt
+- spec/data/illumina/s_1_2_001_qseq.txt
+- spec/data/illumina/s_1_3_001_qseq.txt
+- spec/data/test.fasta
+- spec/data/test.fastq
+- spec/data/test.qseq
+- spec/spec_helper.rb