Basil 0.1.0

data/.rspec

@@ -0,0 +1 @@
+rspec -c -f d

data/.rvmrc

@@ -0,0 +1 @@
+rvm use 1.8.7@basil --create

data/Gemfile

@@ -0,0 +1,7 @@
+source :rubygems
+gemspec
+gem "dna"
+gem "awesome_print"
+gem "trollop"
+gem "rspec"
+gem "awesome_print"

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/basil.gemspec

@@ -0,0 +1,74 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "Basil"
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Austin G. Davis-Richardson"]
+  s.date = "2012-09-18"
+  s.description = "Separate Illumin and 454 sequence output by barcode"
+  s.email = "harekrishna@gmail.com"
+  s.executables = ["basil"]
+  s.files = [
+    ".rspec",
+    ".rvmrc",
+    "Gemfile",
+    "Rakefile",
+    "basil.gemspec",
+    "bin/basil",
+    "lib/basil.rb",
+    "lib/basil/barcodes.rb",
+    "lib/basil/basil.rb",
+    "lib/basil/buffer.rb",
+    "lib/basil/cli.rb",
+    "readme.md",
+    "spec/basil_spec.rb",
+    "spec/buffer_spec.rb",
+    "spec/cli_spec.rb",
+    "spec/data/barcodes.csv",
+    "spec/data/illumina/s_1_1_001_qseq.txt",
+    "spec/data/illumina/s_1_2_001_qseq.txt",
+    "spec/data/illumina/s_1_3_001_qseq.txt",
+    "spec/data/test.fasta",
+    "spec/data/test.fastq",
+    "spec/data/test.qseq",
+    "spec/spec_helper.rb"
+  ]
+  s.homepage = "http://audy.github.com/basil"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.24"
+  s.summary = "HTPS Demultiplexer"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<Basil>, [">= 0"])
+      s.add_runtime_dependency(%q<dna>, [">= 0"])
+      s.add_runtime_dependency(%q<trollop>, [">= 0"])
+      s.add_runtime_dependency(%q<progressbar>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+    else
+      s.add_dependency(%q<Basil>, [">= 0"])
+      s.add_dependency(%q<dna>, [">= 0"])
+      s.add_dependency(%q<trollop>, [">= 0"])
+      s.add_dependency(%q<progressbar>, [">= 0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<Basil>, [">= 0"])
+    s.add_dependency(%q<dna>, [">= 0"])
+    s.add_dependency(%q<trollop>, [">= 0"])
+    s.add_dependency(%q<progressbar>, [">= 0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+  end
+end
+

data/bin/basil

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib', 'basil'))
+
+Basil::CLI.run!

data/lib/basil.rb

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'bundler'
+
+Bundler.require :default
+
+Dir.glob(File.join(File.dirname(__FILE__), 'basil', '*.rb')).each do |f|
+  require f
+end

data/lib/basil/barcodes.rb

@@ -0,0 +1,3 @@
+class Barcodes
+  # TODO add commonly used barcodes where copyright permits.
+end

data/lib/basil/basil.rb

@@ -0,0 +1,60 @@
+module Basil
+
+  class Basil
+
+    #
+    # Gets Basil ready, must specify a barcodes hash with Basil.new barcodes
+    #
+    # Barcodes hash has regexp for values. If not, strings are converted to regexp
+    # Barcode names are keys, used to generate filenames.
+    #
+    def initialize(barcodes)
+      @barcodes = Hash.new
+      barcodes.each_pair do |k, v|
+        @barcodes[k] = Regexp.new "^#{v}", true
+      end
+      @barcodes
+    end
+
+    #
+    # Finds the barcode (if present)
+    #
+    # If the barcode is present, returns 'name',
+    # and the sequence with the barcode removed
+    # If not, returns nil
+    #
+    def recognize(string)
+      matches = @barcodes.each_pair.collect do |k, v|
+        k if string[v]
+      end
+
+      matches.compact!
+
+      if matches.length > 1
+        raise Exception, "sequence #{string} has more than one match"
+      elsif matches.length == 0
+        nil
+      else
+        barcode = matches.first
+        sequence = string.gsub(@barcodes[barcode], '')
+        [barcode, sequence]
+      end
+    end
+
+    #
+    # Parses a barcodes file
+    #
+    # barcodes are specified in CSV file
+    # (unless specified otherwise with :sep => '')
+    # barcode_name,sequence
+    # the barcode_name will be used to generate the filename
+    #
+    def self.parse_barcodes(handle, args={})
+      sep = args[:sep] || ","
+      barcodes = handle.each.collect do |line|
+        name, barcode = line.strip.split(sep)
+      end
+      Hash[*barcodes.flatten]
+    end
+  end
+end

data/lib/basil/buffer.rb

@@ -0,0 +1,50 @@
+module Basil
+  class Buffer
+
+    #
+    # Create a new buffer
+    #
+    def initialize(args={})
+      @buffer = Hash.new { |h, k| h[k] = Array.new }
+      @buffer_max = args[:buffer_max] || 100_000
+    end
+
+    #
+    # Add an object to the buffer
+    #
+    def add_to(bucket, obj)
+
+      @buffer[bucket] << obj
+
+      if @buffer[bucket].length > @buffer_max
+        # write out
+        File.open(bucket, 'a+') do |out|
+          @buffer[bucket].each do |v|
+            out.puts v
+          end
+        end
+
+        # clear that bucket
+        @buffer[bucket].clear
+      end
+    end
+
+    def [](k)
+      @buffer[k]
+    end
+
+    #
+    # Writes out leftover objects
+    #
+    def finalize
+      @buffer.each_key do |bucket|
+        File.open(bucket, 'a+') do |out|
+          @buffer[bucket].each do |v|
+            out.puts v
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/basil/cli.rb

@@ -0,0 +1,102 @@
+module Basil
+  class CLI
+    class << self
+
+    def run!
+
+      opts           = parse_arguments
+      barcodes       = parse_barcodes opts[:barcodes]
+      out_dir        = opts[:out]
+      buffer         = Buffer.new
+      basil          = Basil.new(barcodes)
+      handles        = Array.new
+
+      if opts[:illumina]
+        if pretrim
+          raise "pretrim is not supported with paired-end data"
+        end
+      else
+        reads_handle   = File.open(opts[:reads])
+        handles       << reads_handle
+        records_format = records.format
+        pretrim        = opts[:pretrim] || 0
+        afttrim        = -1*opts[:afttrim] || -1
+      end
+
+      if File.exist? out_dir
+        $stderr.puts "#{out_dir} already exists -- delete or move!"
+        exit
+      else
+        Dir.mkdir(out_dir)
+      end
+
+      records.each do |record|
+        
+        match = basil.recognize record.sequence
+        barcode, sequence = match if match
+
+        unless match
+          barcode = 'unknown'
+          trimmed_sequence = record.sequence
+        end
+        
+        trimmed_sequence = record.sequence[pretrim..afttrim]
+
+        new_record = Fasta.new :name => record.name, :sequence => trimmed_sequence
+        buffer.add_to File.join(out_dir, barcode + ".#{records_format}"), new_record
+      end
+
+      # Finish up
+      buffer.finalize
+      handles.collect { |x| x.close }
+    end
+
+    def parse_barcodes barcodes
+      File.open(barcodes) { |h| Basil.parse_barcodes h }
+    end
+
+    #
+    # For parsing illumina-generated output directories containing
+    # paired QSEQ files
+    #
+    def parse_illumina_directory dir
+      files = Dir[File.join(dir, '*')]
+
+      files.collect do |x|
+        m = File.basename(x).match(/s_(\d)_(\d)_(\d*)_qseq\.txt/)
+        { :lane => m[1], :pair => m[2], :n => m[3], :filename => File.join(dir, m[0]) }
+      end.group_by { |x| x[:n] }
+    end
+
+    def parse_arguments
+      opts = Trollop::options do
+
+        version "PRE-RELEASE"
+        banner <<-EOS
+Basil is an HTPS demultiplexer
+
+Version: #{version}
+
+Usage:
+    basil [options] -i <reads/directory> -o <output directory>
+where [options] are:
+        EOS
+
+        opt :barcodes, 'barcodes definition file', :short => 'b', :type => String
+        opt :reads, 'reads (fastq/fasta/qseq)', :short => 'i', :type => String
+        opt :illumina, 'Illumina output directory', :short => 'd'
+        opt :out_fmt, 'Output format (default: same format as input)', :short => 'f'
+        opt :pretrim, 'trim N nucleotides from start before searching for barcode', :type => Integer
+        opt :out, 'output directory', :short => 'o', :type => String
+      end
+
+      filename = ARGV.shift
+      Trollop::die :barcodes, "must specify barcodes" if opts[:barcodes].nil?
+      Trollop::die :reads, "must specify reads or Illumina directory" if !(opts[:reads] || opts[:illumina])
+      Trollop::die "cannot specify both reads and Illumina directory" if opts[:reads] && opts[:illumina]
+      Trollop::die :out, "must specify output directory" if !opts[:out]
+      opts
+    end
+  end 
+  end
+end

data/readme.md

@@ -0,0 +1,49 @@
+# Basil
+
+Yet another tool for splitting barcoded sequences. It had to be done.
+
+## Installation
+
+You're gonna need Ruby 1.8.7 or better.
+
+    (sudo) gem install basil
+
+## Features
+
+**Demultiplexes multiplexed high-throughput sequencing data for Illumina, Roche 454, IonTorrent, _etc_... sequence data in their native formats**
+
+- Supports unlimited number of barcodes
+- Trimming from left end of read
+- Barcodes provided as simple .csv file
+- Support input filetypes:
+    - Fastq
+    - Fasta
+
+## TODO
+
+- Support regexp in barcodes input.
+
+## Usage
+
+    Usage:
+        basil [options] -i <reads/directory> -o <output directory>
+    where [options] are:
+      --barcodes, -b <s>:   barcodes definition file
+             --reads, -i:   reads (fastq/fasta/qseq)
+          --illumina, -d:   Illumina output directory
+       --pretrim, -p <i>:   trim N nucleotides from start
+       --afttrim, -a <i>:   trim N nucleotides from end
+           --out, -o <s>:   output directory
+           --version, -v:   Print version and exit
+              --help, -h:   Show this message
+
+## Barcodes file
+
+Barcodes are specified in a simple CSV file:
+
+    barcode_name,barcode_sequence
+    my_awesome_barcode,gggggggg
+    my_awesomer_barcode,aaaaaaaa
+    my_awesomest_barcode,cccccccc
+
+(Eventually, I will add built-in support for commonly-used barcodes)

data/spec/basil_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+describe Basil::Basil do
+
+  let (:barcodes) {
+    {
+      1 => 'gggg',
+      2 => 'aaaa',
+      3 => 'tttt',
+      4 => 'cccc',
+    }
+  }
+
+  let (:basil) { Basil::Basil.new barcodes }
+
+  let (:dna) { 'ggggnnnn' }
+
+  it 'can be created' do
+    barcodes.should_not raise_error TypeError
+  end
+
+  it 'raises an error if duplicate barcodes are specified' do
+    Basil::Basil.new({ 1 => 'gggg', 2 => 'gggg' }).should raise_error
+  end
+
+  it '#recognize returns the name of matched barcode' do
+    basil.recognize(dna).first.should == 1
+  end
+
+  it 'accepts regexp for barcodes' do
+    basil = Basil::Basil.new({ 1 => /^.GATC/i})
+    basil.recognize("NGATC").first.should == 1
+  end
+
+  it 'preserves case sensitivity of regexp' do
+    basil = Basil::Basil.new(1 => /^.GATC/)
+    basil.recognize("Ngatc").should be_nil
+  end
+
+  it '#recognize is case insensitive' do
+    basil.recognize(dna.upcase!).first.should == 1
+  end
+
+  it '#recognize ^ is prepended to barcode regexp if strings are provided' do
+    basil.recognize('aggggaaa').should be_nil
+  end
+
+  it 'trims barcode from sequence' do
+    basil.recognize(dna).last.should == 'nnnn'
+  end
+
+  it 'can parse a barcodes.csv file producing a hash' do
+    parsed_barcodes = File.open('spec/data/barcodes.csv') { |h| Basil::Basil.parse_barcodes h }
+    parsed_barcodes.should == { 'a' => 'a'*7, 'g' => 'g'*7, 'c' => 'c'*7, 't' => 't'*7 }
+  end
+
+end

data/spec/buffer_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe Basil::Buffer do
+
+  let (:buffer) { Basil::Buffer.new }
+
+  it 'can be created' do
+    buffer.should_not raise_error TypeError
+  end
+
+  it 'can be given stuff' do
+    buffer['thing'] << 'another thing'
+    buffer['thing'].first.should == 'another thing'
+  end
+end

data/spec/cli_spec.rb

@@ -0,0 +1,28 @@
+describe Basil::CLI do
+
+  it 'should be executable' do
+    `bin/basil 2> /dev/null`
+    $?.should_not == 0
+  end
+
+  it 'should have a help menu' do
+    `bin/basil --help`.include?('help').should be_true
+  end
+
+  it 'should parse barcodes.csv files' do
+    barcodes = Basil::CLI.parse_barcodes 'spec/data/barcodes.csv'
+    barcodes.should == { "a" => "aaaaaaa", "g" => "ggggggg", "c" => "ccccccc", "t" => "ttttttt"}
+  end
+
+  it 'can parse an illumina output directory' do
+    files = Basil::CLI.parse_illumina_directory 'spec/data/illumina'
+    directory = {
+      "001" => [
+        {:lane=>"1", :pair=>"1", :n=>"001", :filename=>"spec/data/illumina/s_1_1_001_qseq.txt"},
+        {:lane=>"1", :pair=>"2", :n=>"001", :filename=>"spec/data/illumina/s_1_2_001_qseq.txt"},
+        {:lane=>"1", :pair=>"3", :n=>"001", :filename=>"spec/data/illumina/s_1_3_001_qseq.txt"}
+      ]
+    }
+    files.should == directory
+  end
+end

data/spec/data/barcodes.csv

@@ -0,0 +1,4 @@
+a,aaaaaaa
+g,ggggggg
+c,ccccccc
+t,ttttttt

data/spec/data/illumina/s_1_1_001_qseq.txt

@@ -0,0 +1,5 @@
+HWUSI-EAS1700R  5       4       1       975     930     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1051    933     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1233    932     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1266    945     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1373    940     0       2       ....... BBBBBBB 0

data/spec/data/illumina/s_1_2_001_qseq.txt

@@ -0,0 +1,5 @@
+HWUSI-EAS1700R  5       4       1       975     930     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1051    933     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1233    932     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1266    945     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1373    940     0       2       ....... BBBBBBB 0

data/spec/data/illumina/s_1_3_001_qseq.txt

@@ -0,0 +1,5 @@
+HWUSI-EAS1700R  5       4       1       975     930     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1051    933     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1233    932     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1266    945     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1373    940     0       2       ....... BBBBBBB 0

data/spec/data/test.fasta

@@ -0,0 +1,20 @@
+>1
+AAAAAA
+>2
+GGGGGG
+>3
+TTTTTT
+>4
+CCCCCC
+>5
+AAATTT
+>6
+TTTAAA
+>7
+GGGCCC
+>8
+CCCGGG
+>9
+GAAAAA
+>10
+CAAAAA

data/spec/data/test.fastq

@@ -0,0 +1,20 @@
+@a
+aaaaaaa
++a
+BBBBBBB
+@b
+ttttttt
++b
+BBBBBBB
+@c
+ccccccc
+@c
+BBBBBBB
+@d
+ggggggg
+@d
+BBBBBBB
+@e
+gagagag
+@e
+BBBBBBB

data/spec/data/test.qseq

@@ -0,0 +1,5 @@
+HWUSI-EAS1700R  5       4       1       975     930     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1051    933     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1233    932     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1266    945     0       2       ....... BBBBBBB 0
+HWUSI-EAS1700R  5       4       1       1373    940     0       2       ....... BBBBBBB 0

data/spec/spec_helper.rb

@@ -0,0 +1,3 @@
+$LOAD_PATH << File.join(File.dirname(__FILE__), 'lib')
+
+require 'basil'

metadata

@@ -0,0 +1,170 @@
+--- !ruby/object:Gem::Specification 
+name: Basil
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Austin G. Davis-Richardson
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-09-18 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: Basil
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: dna
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: trollop
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: progressbar
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: jeweler
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: jeweler
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id006
+description: Separate Illumin and 454 sequence output by barcode
+email: harekrishna@gmail.com
+executables: 
+- basil
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .rspec
+- .rvmrc
+- Gemfile
+- Rakefile
+- basil.gemspec
+- bin/basil
+- lib/basil.rb
+- lib/basil/barcodes.rb
+- lib/basil/basil.rb
+- lib/basil/buffer.rb
+- lib/basil/cli.rb
+- readme.md
+- spec/basil_spec.rb
+- spec/buffer_spec.rb
+- spec/cli_spec.rb
+- spec/data/barcodes.csv
+- spec/data/illumina/s_1_1_001_qseq.txt
+- spec/data/illumina/s_1_2_001_qseq.txt
+- spec/data/illumina/s_1_3_001_qseq.txt
+- spec/data/test.fasta
+- spec/data/test.fastq
+- spec/data/test.qseq
+- spec/spec_helper.rb
+homepage: http://audy.github.com/basil
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: HTPS Demultiplexer
+test_files: []
+