ngs_server 0.1

data/ext/bamtools/src/toolkit/bamtools_resolve.h

@@ -0,0 +1,42 @@
+// ***************************************************************************
+// bamtools_resolve.h (c) 2011 Derek Barnett
+// Marth Lab, Department of Biology, Boston College
+// ---------------------------------------------------------------------------
+// Last modified: 23 June 2011
+// ---------------------------------------------------------------------------
+// Resolves paired-end reads (marking the IsProperPair flag as needed).
+// ***************************************************************************
+
+#ifndef BAMTOOLS_RESOLVE_H
+#define BAMTOOLS_RESOLVE_H
+
+#include "bamtools_tool.h"
+
+namespace BamTools {
+
+class ResolveTool : public AbstractTool {
+
+    public:
+        ResolveTool(void);
+        ~ResolveTool(void);
+
+    public:
+        int Help(void);
+        int Run(int argc, char* argv[]);
+
+    private:
+        struct ResolveSettings;
+        ResolveSettings* m_settings;
+
+        struct ResolveToolPrivate;
+        ResolveToolPrivate* m_impl;
+
+        struct ReadNamesFileReader;
+        struct ReadNamesFileWriter;
+        struct StatsFileReader;
+        struct StatsFileWriter;
+};
+
+} // namespace BamTools
+
+#endif // BAMTOOLS_RESOLVE_H

data/ext/bamtools/src/toolkit/bamtools_revert.cpp

@@ -0,0 +1,194 @@
+// ***************************************************************************
+// bamtools_revert.cpp (c) 2010 Derek Barnett, Alistair Ward
+// Marth Lab, Department of Biology, Boston College
+// ---------------------------------------------------------------------------
+// Last modified: 7 April 2011
+// ---------------------------------------------------------------------------
+// Removes duplicate marks and restores original base qualities
+// ***************************************************************************
+
+#include "bamtools_revert.h"
+
+#include <api/BamReader.h>
+#include <api/BamWriter.h>
+#include <utils/bamtools_options.h>
+#include <utils/bamtools_utilities.h>
+using namespace BamTools;
+
+#include <iostream>
+#include <string>
+using namespace std;
+
+namespace BamTools {
+
+static const string OQ_TAG = "OQ";
+
+} // namespace BamTools;
+
+// ---------------------------------------------
+// RevertSettings implementation
+
+struct RevertTool::RevertSettings {
+
+    // flags
+    bool HasInput;
+    bool HasOutput;
+    bool IsForceCompression;
+    bool IsKeepDuplicateFlag;
+    bool IsKeepQualities;
+
+    // filenames
+    string InputFilename;
+    string OutputFilename;
+    
+    // constructor
+    RevertSettings(void)
+        : HasInput(false)
+        , HasOutput(false)
+        , IsForceCompression(false)
+        , IsKeepDuplicateFlag(false)
+        , IsKeepQualities(false)
+        , InputFilename(Options::StandardIn())
+        , OutputFilename(Options::StandardOut())
+    { }
+};  
+
+// ---------------------------------------------
+// RevertToolPrivate implementation
+
+struct RevertTool::RevertToolPrivate {
+  
+    // ctor & dtor
+    public:
+        RevertToolPrivate(RevertTool::RevertSettings* settings)
+            : m_settings(settings)
+        { }
+        ~RevertToolPrivate(void) { }
+  
+    // 'public' interface
+    public:
+        bool Run(void);
+        
+    // internal methods
+    private:
+        void RevertAlignment(BamAlignment& al);
+        
+    // data members
+    private:
+        RevertTool::RevertSettings* m_settings;
+};
+
+// 'reverts' a BAM alignment
+// default behavior (for now) is:
+//   1 - replace Qualities with OQ contents
+//   2 - clear IsDuplicate flag
+// can override default behavior using command line options
+void RevertTool::RevertToolPrivate::RevertAlignment(BamAlignment& al) {
+
+    // replace Qualities with OQ contents, if requested
+    if ( !m_settings->IsKeepQualities ) {
+        string originalQualities;
+        if ( al.GetTag(OQ_TAG, originalQualities) ) {
+            al.Qualities = originalQualities;
+            al.RemoveTag(OQ_TAG);
+        }
+    }
+
+    // clear duplicate flag, if requested
+    if ( !m_settings->IsKeepDuplicateFlag )
+        al.SetIsDuplicate(false);
+}
+
+bool RevertTool::RevertToolPrivate::Run(void) {
+  
+    // opens the BAM file without checking for indexes
+    BamReader reader;
+    if ( !reader.Open(m_settings->InputFilename) ) {
+        cerr << "bamtools revert ERROR: could not open " << m_settings->InputFilename
+             << " for reading... Aborting." << endl;
+        return false;
+    }
+
+    // get BAM file metadata
+    const string& headerText = reader.GetHeaderText();
+    const RefVector& references = reader.GetReferenceData();
+    
+    // determine compression mode for BamWriter
+    bool writeUncompressed = ( m_settings->OutputFilename == Options::StandardOut() &&
+                              !m_settings->IsForceCompression );
+    BamWriter::CompressionMode compressionMode = BamWriter::Compressed;
+    if ( writeUncompressed ) compressionMode = BamWriter::Uncompressed;
+
+    // open BamWriter
+    BamWriter writer;
+    writer.SetCompressionMode(compressionMode);
+    if ( !writer.Open(m_settings->OutputFilename, headerText, references) ) {
+        cerr << "bamtools revert ERROR: could not open " << m_settings->OutputFilename
+             << " for writing... Aborting." << endl;
+        reader.Close();
+        return false;
+    }
+
+    // plow through file, reverting alignments
+    BamAlignment al;
+    while ( reader.GetNextAlignment(al) ) {
+        RevertAlignment(al);
+        writer.SaveAlignment(al);
+    }
+    
+    // clean and exit
+    reader.Close();
+    writer.Close();
+    return true; 
+}
+
+// ---------------------------------------------
+// RevertTool implementation
+
+RevertTool::RevertTool(void)
+    : AbstractTool()
+    , m_settings(new RevertSettings)
+    , m_impl(0)
+{
+    // set program details
+    Options::SetProgramInfo("bamtools revert", "removes duplicate marks and restores original (non-recalibrated) base qualities", "[-in <filename> -in <filename> ...] [-out <filename> | [-forceCompression]] [revertOptions]");
+    
+    // set up options 
+    OptionGroup* IO_Opts = Options::CreateOptionGroup("Input & Output");
+    Options::AddValueOption("-in",  "BAM filename", "the input BAM file",  "", m_settings->HasInput,  m_settings->InputFilename,  IO_Opts, Options::StandardIn());
+    Options::AddValueOption("-out", "BAM filename", "the output BAM file", "", m_settings->HasOutput, m_settings->OutputFilename, IO_Opts, Options::StandardOut());
+    Options::AddOption("-forceCompression", "if results are sent to stdout (like when piping to another tool), default behavior is to leave output uncompressed. Use this flag to override and force compression", m_settings->IsForceCompression, IO_Opts);
+
+    OptionGroup* RevertOpts = Options::CreateOptionGroup("Revert Options");
+    Options::AddOption("-keepDuplicate", "keep duplicates marked", m_settings->IsKeepDuplicateFlag, RevertOpts);
+    Options::AddOption("-keepQualities", "keep base qualities (do not replace with OQ contents)", m_settings->IsKeepQualities, RevertOpts);
+}
+
+RevertTool::~RevertTool(void) {
+
+    delete m_settings;
+    m_settings = 0;
+    
+    delete m_impl;
+    m_impl = 0;
+}
+
+int RevertTool::Help(void) {
+    Options::DisplayHelp();
+    return 0;
+}
+
+int RevertTool::Run(int argc, char* argv[]) {
+  
+    // parse command line arguments
+    Options::Parse(argc, argv, 1);
+
+    // intialize RevertTool with settings
+    m_impl = new RevertToolPrivate(m_settings);
+    
+    // run RevertTool, return success/fail
+    if ( m_impl->Run() )
+        return 0;
+    else
+        return 1;
+}

data/ext/bamtools/src/toolkit/bamtools_revert.h

@@ -0,0 +1,37 @@
+// ***************************************************************************
+// bamtools_revert.h (c) 2010 Derek Barnett, Alistair Ward
+// Marth Lab, Department of Biology, Boston College
+// ---------------------------------------------------------------------------
+// Last modified: 7 April 2011
+// ---------------------------------------------------------------------------
+// Removes duplicate marks and restores original base qualities
+// ***************************************************************************
+
+#ifndef BAMTOOLS_REVERT_H
+#define BAMTOOLS_REVERT_H
+
+#include "bamtools_tool.h"
+
+namespace BamTools {
+  
+class RevertTool : public AbstractTool {
+  
+    public:
+        RevertTool(void);
+        ~RevertTool(void);
+  
+    public:
+        int Help(void);
+        int Run(int argc, char* argv[]); 
+        
+    private:
+        struct RevertSettings;
+        RevertSettings* m_settings;
+        
+        struct RevertToolPrivate;
+        RevertToolPrivate* m_impl;
+};
+  
+} // namespace BamTools
+
+#endif // BAMTOOLS_REVERT_H

data/ext/bamtools/src/toolkit/bamtools_sort.cpp

@@ -0,0 +1,410 @@
+// ***************************************************************************
+// bamtools_sort.cpp (c) 2010 Derek Barnett, Erik Garrison
+// Marth Lab, Department of Biology, Boston College
+// ---------------------------------------------------------------------------
+// Last modified: 14 June 2011 (DB)
+// ---------------------------------------------------------------------------
+// Sorts an input BAM file
+// ***************************************************************************
+
+#include "bamtools_sort.h"
+
+#include <api/SamConstants.h>
+#include <api/BamMultiReader.h>
+#include <api/BamWriter.h>
+#include <utils/bamtools_options.h>
+using namespace BamTools;
+
+#include <cstdio>
+#include <algorithm>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+using namespace std;
+
+namespace BamTools {
+  
+// defaults
+//
+// ** These defaults should be tweaked & 'optimized' per testing ** //
+//
+//    I say 'optimized' because each system will naturally perform
+//    differently.  We will attempt to determine a sensible
+//    compromise that should perform well on average.
+const unsigned int SORT_DEFAULT_MAX_BUFFER_COUNT  = 500000;  // max numberOfAlignments for buffer
+const unsigned int SORT_DEFAULT_MAX_BUFFER_MEMORY = 1024;    // Mb
+
+// -----------------------------------
+// comparison objects (for sorting)
+
+struct SortLessThanPosition {
+    bool operator() (const BamAlignment& lhs, const BamAlignment& rhs) {
+        if ( lhs.RefID != rhs.RefID )
+            return lhs.RefID < rhs.RefID;
+        else
+            return lhs.Position < rhs.Position;
+    }
+};
+
+struct SortLessThanName {
+    bool operator() (const BamAlignment& lhs, const BamAlignment& rhs) {
+        return lhs.Name < rhs.Name;
+    }
+};
+    
+} // namespace BamTools
+
+// ---------------------------------------------
+// SortSettings implementation
+
+struct SortTool::SortSettings {
+
+    // flags
+    bool HasInputBamFilename;
+    bool HasMaxBufferCount;
+    bool HasMaxBufferMemory;
+    bool HasOutputBamFilename;
+    bool IsSortingByName;
+
+    // filenames
+    string InputBamFilename;
+    string OutputBamFilename;
+
+    // parameters
+    unsigned int MaxBufferCount;
+    unsigned int MaxBufferMemory;
+
+    // constructor
+    SortSettings(void)
+        : HasInputBamFilename(false)
+        , HasMaxBufferCount(false)
+        , HasMaxBufferMemory(false)
+        , HasOutputBamFilename(false)
+        , IsSortingByName(false)
+        , InputBamFilename(Options::StandardIn())
+        , OutputBamFilename(Options::StandardOut())
+        , MaxBufferCount(SORT_DEFAULT_MAX_BUFFER_COUNT)
+        , MaxBufferMemory(SORT_DEFAULT_MAX_BUFFER_MEMORY)
+    { }
+};
+
+// ---------------------------------------------
+// SortToolPrivate implementation
+
+class SortTool::SortToolPrivate {
+      
+    // ctor & dtor
+    public:
+        SortToolPrivate(SortTool::SortSettings* settings);
+        ~SortToolPrivate(void) { }
+        
+    // 'public' interface
+    public:
+        bool Run(void);
+        
+    // internal methods
+    private:
+        bool CreateSortedTempFile(vector<BamAlignment>& buffer);
+        bool GenerateSortedRuns(void);
+        bool MergeSortedRuns(void);
+        bool WriteTempFile(const vector<BamAlignment>& buffer, const string& tempFilename);
+        void SortBuffer(vector<BamAlignment>& buffer);
+        
+    // data members
+    private:
+        SortTool::SortSettings* m_settings;
+        string m_tempFilenameStub;
+        int m_numberOfRuns;
+        string m_headerText;
+        RefVector m_references;
+        vector<string> m_tempFilenames;
+};
+
+// constructor
+SortTool::SortToolPrivate::SortToolPrivate(SortTool::SortSettings* settings) 
+    : m_settings(settings)
+    , m_numberOfRuns(0) 
+{ 
+    // set filename stub depending on inputfile path
+    // that way multiple sort runs don't trip on each other's temp files
+    if ( m_settings) {
+        size_t extensionFound = m_settings->InputBamFilename.find(".bam");
+        if (extensionFound != string::npos )
+            m_tempFilenameStub = m_settings->InputBamFilename.substr(0,extensionFound);
+        m_tempFilenameStub.append(".sort.temp.");
+    }
+}
+
+// generates mutiple sorted temp BAM files from single unsorted BAM file
+bool SortTool::SortToolPrivate::GenerateSortedRuns(void) {
+    
+    // open input BAM file
+    BamReader reader;
+    if ( !reader.Open(m_settings->InputBamFilename) ) {
+        cerr << "bamtools sort ERROR: could not open " << m_settings->InputBamFilename
+             << " for reading... Aborting." << endl;
+        return false;
+    }
+    
+    // get basic data that will be shared by all temp/output files 
+    SamHeader header = reader.GetHeader();
+    header.SortOrder = ( m_settings->IsSortingByName
+                       ? Constants::SAM_HD_SORTORDER_QUERYNAME
+                       : Constants::SAM_HD_SORTORDER_COORDINATE );
+    m_headerText = header.ToString();
+    m_references = reader.GetReferenceData();
+    
+    // set up alignments buffer
+    BamAlignment al;
+    vector<BamAlignment> buffer;
+    buffer.reserve( (size_t)(m_settings->MaxBufferCount*1.1) );
+    bool bufferFull = false;
+    
+    // if sorting by name, we need to generate full char data
+    // so can't use GetNextAlignmentCore()
+    if ( m_settings->IsSortingByName ) {
+
+        // iterate through file
+        while ( reader.GetNextAlignment(al)) {
+
+            // check buffer's usage
+            bufferFull = ( buffer.size() >= m_settings->MaxBufferCount );
+
+            // store alignments until buffer is "full"
+            if ( !bufferFull )
+                buffer.push_back(al);
+
+            // if buffer is "full"
+            else {
+
+                // push any unmapped reads into buffer,
+                // don't want to split these into a separate temp file
+                if ( !al.IsMapped() )
+                    buffer.push_back(al);
+
+                // "al" is mapped, so create a sorted temp file with current buffer contents
+                // then push "al" into fresh buffer
+                else {
+                    CreateSortedTempFile(buffer);
+                    buffer.push_back(al);
+                }
+            }
+        }
+    }
+
+    // sorting by position, can take advantage of GNACore() speedup
+    else {
+
+        // iterate through file
+        while ( reader.GetNextAlignmentCore(al) ) {
+
+            // check buffer's usage
+            bufferFull = ( buffer.size() >= m_settings->MaxBufferCount );
+
+            // store alignments until buffer is "full"
+            if ( !bufferFull )
+                buffer.push_back(al);
+
+            // if buffer is "full"
+            else {
+
+                // push any unmapped reads into buffer,
+                // don't want to split these into a separate temp file
+                if ( !al.IsMapped() )
+                    buffer.push_back(al);
+
+                // "al" is mapped, so create a sorted temp file with current buffer contents
+                // then push "al" into fresh buffer
+                else {
+                    CreateSortedTempFile(buffer);
+                    buffer.push_back(al);
+                }
+            }
+        }
+    }
+
+    // handle any leftover buffer contents
+    if ( !buffer.empty() )
+        CreateSortedTempFile(buffer);
+    
+    // close reader & return success
+    reader.Close();
+    return true;
+}
+
+bool SortTool::SortToolPrivate::CreateSortedTempFile(vector<BamAlignment>& buffer) {
+ 
+    // do sorting
+    SortBuffer(buffer);
+  
+    // write sorted contents to temp file, store success/fail
+    stringstream tempStr;
+    tempStr << m_tempFilenameStub << m_numberOfRuns;
+    bool success = WriteTempFile( buffer, tempStr.str() );
+    
+    // save temp filename for merging later
+    m_tempFilenames.push_back(tempStr.str());
+    
+    // clear buffer contents & update run counter
+    buffer.clear();
+    ++m_numberOfRuns;
+    
+    // return success/fail of writing to temp file
+    // TODO: a failure returned here is not actually caught and handled anywhere
+    return success;
+}
+
+// merges sorted temp BAM files into single sorted output BAM file
+bool SortTool::SortToolPrivate::MergeSortedRuns(void) {
+  
+    // open up multi reader for all of our temp files
+    // this might get broken up if we do a multi-pass system later ??
+    BamMultiReader multiReader;
+    if ( !multiReader.Open(m_tempFilenames) ) {
+        cerr << "bamtools sort ERROR: could not open BamMultiReader for merging temp files... Aborting."
+             << endl;
+        return false;
+    }
+
+    // set sort order for merge
+    if ( m_settings->IsSortingByName )
+        multiReader.SetSortOrder(BamMultiReader::SortedByReadName);
+    else
+        multiReader.SetSortOrder(BamMultiReader::SortedByPosition);
+    
+    // open writer for our completely sorted output BAM file
+    BamWriter mergedWriter;
+    if ( !mergedWriter.Open(m_settings->OutputBamFilename, m_headerText, m_references) ) {
+        cerr << "bamtools sort ERROR: could not open " << m_settings->OutputBamFilename
+             << " for writing... Aborting." << endl;
+        multiReader.Close();
+        return false;
+    }
+    
+    // while data available in temp files
+    BamAlignment al;
+    while ( multiReader.GetNextAlignmentCore(al) )
+        mergedWriter.SaveAlignment(al);
+  
+    // close readers
+    multiReader.Close();
+    mergedWriter.Close();
+    
+    // delete all temp files
+    vector<string>::const_iterator tempIter = m_tempFilenames.begin();
+    vector<string>::const_iterator tempEnd  = m_tempFilenames.end();
+    for ( ; tempIter != tempEnd; ++tempIter ) {
+        const string& tempFilename = (*tempIter);
+        remove(tempFilename.c_str());
+    }
+  
+    return true;
+}
+
+bool SortTool::SortToolPrivate::Run(void) {
+ 
+    // this does a single pass, chunking up the input file into smaller sorted temp files, 
+    // then write out using BamMultiReader to handle merging
+    
+    if ( GenerateSortedRuns() )
+        return MergeSortedRuns();
+    else 
+        return false;
+} 
+    
+void SortTool::SortToolPrivate::SortBuffer(vector<BamAlignment>& buffer) {
+ 
+    // ** add further custom sort options later ?? **
+    
+    // sort buffer by desired method
+    if ( m_settings->IsSortingByName )
+        std::stable_sort ( buffer.begin(), buffer.end(), SortLessThanName() );
+    else 
+        std::stable_sort ( buffer.begin(), buffer.end(), SortLessThanPosition() );
+}
+    
+bool SortTool::SortToolPrivate::WriteTempFile(const vector<BamAlignment>& buffer,
+                                              const string& tempFilename)
+{
+    // open temp file for writing
+    BamWriter tempWriter;
+    if ( !tempWriter.Open(tempFilename, m_headerText, m_references) ) {
+        cerr << "bamtools sort ERROR: could not open " << tempFilename
+             << " for writing." << endl;
+        return false;
+    }
+  
+    // write data
+    vector<BamAlignment>::const_iterator buffIter = buffer.begin();
+    vector<BamAlignment>::const_iterator buffEnd  = buffer.end();
+    for ( ; buffIter != buffEnd; ++buffIter )  {
+        const BamAlignment& al = (*buffIter);
+        tempWriter.SaveAlignment(al);
+    }
+  
+    // close temp file & return success
+    tempWriter.Close();
+    return true;
+}
+
+// ---------------------------------------------
+// SortTool implementation
+
+SortTool::SortTool(void)
+    : AbstractTool()
+    , m_settings(new SortSettings)
+    , m_impl(0)
+{
+    // set program details
+    Options::SetProgramInfo("bamtools sort", "sorts a BAM file", "[-in <filename>] [-out <filename>] [sortOptions]");
+
+    // set up options
+    OptionGroup* IO_Opts = Options::CreateOptionGroup("Input & Output");
+    Options::AddValueOption("-in",  "BAM filename", "the input BAM file",  "",
+                            m_settings->HasInputBamFilename,  m_settings->InputBamFilename,
+                            IO_Opts, Options::StandardIn());
+    Options::AddValueOption("-out", "BAM filename", "the output BAM file", "",
+                            m_settings->HasOutputBamFilename, m_settings->OutputBamFilename,
+                            IO_Opts, Options::StandardOut());
+
+    OptionGroup* SortOpts = Options::CreateOptionGroup("Sorting Methods");
+    Options::AddOption("-byname", "sort by alignment name", m_settings->IsSortingByName, SortOpts);
+
+    OptionGroup* MemOpts = Options::CreateOptionGroup("Memory Settings");
+    Options::AddValueOption("-n",   "count", "max number of alignments per tempfile", "",
+                            m_settings->HasMaxBufferCount,  m_settings->MaxBufferCount,
+                            MemOpts, SORT_DEFAULT_MAX_BUFFER_COUNT);
+    Options::AddValueOption("-mem", "Mb", "max memory to use", "",
+                            m_settings->HasMaxBufferMemory, m_settings->MaxBufferMemory,
+                            MemOpts, SORT_DEFAULT_MAX_BUFFER_MEMORY);
+}
+
+SortTool::~SortTool(void) {
+
+    delete m_settings;
+    m_settings = 0;
+
+    delete m_impl;
+    m_impl = 0;
+}
+
+int SortTool::Help(void) {
+    Options::DisplayHelp();
+    return 0;
+}
+
+int SortTool::Run(int argc, char* argv[]) {
+
+    // parse command line arguments
+    Options::Parse(argc, argv, 1);
+
+    // initialize SortTool with settings
+    m_impl = new SortToolPrivate(m_settings);
+
+    // run SortTool, return success/fail
+    if ( m_impl->Run() )
+        return 0;
+    else
+        return 1;
+}