bio-samtools 0.2.0

data/README.rdoc

@@ -0,0 +1,85 @@
+= bio-samtools
+
+The original project samtools-ruby belongs to Ricardo H. Ramirez @ https://github.com/homonecloco/samtools-ruby
+
+== Introduction
+
+
+Documentation and code come from that project and we'll adapt it for a better integration in BioRuby.
+
+Binder of samtools for ruby, on the top of FFI. 
+
+This project was born from the need to add support of BAM files to 
+the gee_fu genome browser (http://github.com/danmaclean/gee_fu). 
+
+== Installation
+At the moment, the only way to "install" the module is to copy it to copy the content 
+of the lib folder on any lib folder you have. You must also copy your libbam.a (linux)
+or libbam.dll (windows) from samtools. If you have a Mac, add the following variable
+to the makefile of samtools
+
+DYFLAGS 	= -dynamiclib -lz
+
+and the following target:
+dylib: libbam.dylib
+libbam.dylib:$(LOBJS)
+		$(CC) $(CFLAGS) $(DYFLAGS) -o libbam.dylib $(LOBJS)
+
+Then, run 
+
+make dylib
+
+Finally, you copy the .dylib file to your lib folder. 
+
+== Usage
+The easiest way to see in "action" samtools-ruby to call
+rake test. 
+
+
+== Dependencies:
+-FFI (http://github.com/ffi/ffi)
+-libbam
+	This can be obtained from samtools.  (http://samtools.sourceforge.net/ )
+	
+== FAQ.
+I´m getting a segmentation Fault, what did I do wrong?
+There are two known segmentation faults at the moment
+	-When you try to load a text file as binary file
+	-When you try to lad a binary file as a text file
+
+If this is not the problem, or you have any other question,  don´t hesitate on 
+dropping a line to
+Ricardo dot Ramirez-Gonzalez at bbsrc dot ac dot uk
+
+== TODO
+-Write a gem to install it
+-Filter to the fetching algorithm (give a condition that has to be satisfied to add the alignment to the list)
+-Examples of how to use it, besides the test folder
+-Operating system independent, DONE ( test needed)
+
+== IMPORTANT NOTE
+-Libraries are compiled for 64 bit machines
+-The gem/archive comes with preinstalled library .a and dylib, .dll is missing I didn't compile it.
+-Library .a, I didn't check if it works on linux (test needed) 
+-Library libbam.so.1, The library is simply compiled with make dylib without any special parameter on
+CentOS5.5 x86_64, gcc (GCC) 4.1.2 20080704 (Red Hat 4.1.2-48). 
+$ openssl dgst libbam.so.1
+MD5(libbam.so.1)= c45cfccfb41ffeb2730ee4b227d244c4
+
+
+
+== Contributing to bio-samtools
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Raoul J.P. Bonnal. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,59 @@
+require 'rubygems'
+require 'bundler'
+
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "bio-samtools"
+  gem.homepage = "http://github.com/helios/bioruby-samtools"
+  gem.license = "MIT"
+  gem.summary = %Q{Binder of samtools for ruby, on the top of FFI.}
+  gem.description = %Q{Binder of samtools for ruby, on the top of FFI. 
+
+  This project was born from the need to add support of BAM files to 
+  the gee_fu genome browser (http://github.com/danmaclean/gee_fu).}
+  gem.email = "ilpuccio.febo@gmail.com"
+  gem.authors = ["Ricardo Ramirez-Gonzalez","Dan MacLean","Raoul J.P. Bonnal"]
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+  gem.extensions = "ext/mkrf_conf.rb"
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "bio-samtools #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+0.2.0

data/bio-samtools.gemspec

@@ -0,0 +1,105 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{bio-samtools}
+  s.version = "0.2.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Ricardo Ramirez-Gonzalez", "Dan MacLean", "Raoul J.P. Bonnal"]
+  s.date = %q{2011-05-23}
+  s.description = %q{Binder of samtools for ruby, on the top of FFI. 
+
+  This project was born from the need to add support of BAM files to 
+  the gee_fu genome browser (http://github.com/danmaclean/gee_fu).}
+  s.email = %q{ilpuccio.febo@gmail.com}
+  s.extensions = ["ext/mkrf_conf.rb"]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "bio-samtools.gemspec",
+    "ext/mkrf_conf.rb",
+    "lib/bio-samtools.rb",
+    "lib/bio/.DS_Store",
+    "lib/bio/db/sam.rb",
+    "lib/bio/db/sam/bam.rb",
+    "lib/bio/db/sam/external/COPYING",
+    "lib/bio/db/sam/external/VERSION",
+    "lib/bio/db/sam/faidx.rb",
+    "lib/bio/db/sam/library.rb",
+    "lib/bio/db/sam/sam.rb",
+    "test/basictest.rb",
+    "test/coverage.rb",
+    "test/coverage_plot.rb",
+    "test/feature.rb",
+    "test/helper.rb",
+    "test/samples/small/ids2.txt",
+    "test/samples/small/sorted.bam",
+    "test/samples/small/test",
+    "test/samples/small/test.bam",
+    "test/samples/small/test.fa",
+    "test/samples/small/test.fai",
+    "test/samples/small/test.sai",
+    "test/samples/small/test.tam",
+    "test/samples/small/test_chr.fasta",
+    "test/samples/small/test_chr.fasta.amb",
+    "test/samples/small/test_chr.fasta.ann",
+    "test/samples/small/test_chr.fasta.bwt",
+    "test/samples/small/test_chr.fasta.fai",
+    "test/samples/small/test_chr.fasta.pac",
+    "test/samples/small/test_chr.fasta.rbwt",
+    "test/samples/small/test_chr.fasta.rpac",
+    "test/samples/small/test_chr.fasta.rsa",
+    "test/samples/small/test_chr.fasta.sa",
+    "test/samples/small/testu.bam",
+    "test/samples/small/testu.bam.bai",
+    "test/test_bio-samtools.rb"
+  ]
+  s.homepage = %q{http://github.com/helios/bioruby-samtools}
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.5.0}
+  s.summary = %q{Binder of samtools for ruby, on the top of FFI.}
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<ffi>, [">= 0"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<bio>, [">= 1.4.1"])
+      s.add_development_dependency(%q<ffi>, [">= 0"])
+    else
+      s.add_dependency(%q<ffi>, [">= 0"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<bio>, [">= 1.4.1"])
+      s.add_dependency(%q<ffi>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<ffi>, [">= 0"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<bio>, [">= 1.4.1"])
+    s.add_dependency(%q<ffi>, [">= 0"])
+  end
+end
+

data/ext/mkrf_conf.rb

@@ -0,0 +1,65 @@
+#(c) Copyright 2011 Raoul Bonnal. All Rights Reserved. 
+
+# create Rakefile for shared library compilation
+
+
+
+path = File.expand_path(File.dirname(__FILE__))
+
+path_external = File.join(path, "../lib/bio/db/sam/external")
+
+version = File.open(File.join(path_external,"VERSION"),'r')
+Version = version.read
+version.close
+
+url = "http://sourceforge.net/projects/samtools/files/samtools/#{Version}/samtools-#{Version}.tar.bz2/download"
+SamToolsFile = "samtools-#{Version}.tar.bz2"
+
+File.open(File.join(path,"Rakefile"),"w") do |rakefile|
+rakefile.write <<-RAKE
+require 'rbconfig'
+require 'open-uri'
+require 'fileutils'
+include FileUtils::Verbose
+require 'rake/clean'
+
+URL = "#{url}"
+
+task :download do
+  open(URL) do |uri|
+    File.open("#{SamToolsFile}",'wb') do |fout|
+      fout.write(uri.read)
+    end #fout 
+  end #uri
+end
+    
+task :compile do
+  sh "tar xvfj #{SamToolsFile}"
+  cd("samtools-#{Version}") do
+    sh "make"
+    cp("libbam.a","#{path_external}")
+    case Config::CONFIG['host_os']
+      when /linux/
+        sh "make libbam.so.1-local"
+        cp("libbam.so.1","#{path_external}")
+      when /darwin/
+        sh "make libbam.1.dylib-local"
+        cp("libbam.1.dylib","#{path_external}")      
+      when /mswin|mingw/ then raise NotImplementedError, "BWA library is not available for Windows platform"  
+    end#case
+  end #cd
+end
+  
+task :clean do
+  cd("samtools-#{Version}") do
+    sh "make clean"
+  end
+  rm("#{SamToolsFile}")
+  rm_rf("samtools-#{Version}")
+end
+
+task :default => [:download, :compile, :clean]
+  
+RAKE
+  
+end

data/lib/bio-samtools.rb

@@ -0,0 +1,2 @@
+require 'ffi'
+require 'bio/db/sam'

data/lib/bio/db/sam.rb

@@ -0,0 +1,325 @@
+require 'bio/db/sam/library'
+require 'bio/db/sam/bam'
+require 'bio/db/sam/faidx'
+require 'bio/db/sam/sam'
+
+module LibC
+  extend FFI::Library
+  ffi_lib FFI::Library::LIBC
+  attach_function :free, [ :pointer ], :void
+  # call #attach_function to attach to malloc, free, memcpy, bcopy, etc.
+end
+
+module Bio
+  class DB
+    class Sam
+      attr_reader :sam_file
+
+      def initialize(optsa={})
+        opts =  { :fasta => nil,  :bam => nil,:tam => nil, :compressed => true, :write => false }.merge!(optsa)
+
+
+
+        @fasta_path = opts[:fasta]
+        @compressed = opts[:compressed]
+        @write = opts[:write]
+        bam = opts[:bam]
+        tam = opts[:tam]
+
+        if bam == nil && tam == nil && @fasta_path == nil then
+          raise SAMException.new(), "No alignment or reference file"
+        elsif bam != nil && tam != nil then
+          raise SAMException.new(), "Alignment has to be in either text or binary format, not both"
+        elsif bam != nil then
+          @binary = true
+          @sam = bam     
+        elsif tam != nil then
+          @sam = tam     
+          @binary = false
+
+        end
+        @fasta_file = nil
+        @sam_file   = nil
+
+        ObjectSpace.define_finalizer(self,  self.class.method(:finalize).to_proc)
+      end
+
+      def open()
+
+        raise SAMException.new(), "Writing not supported yet" if @write
+        raise SAMException.new(), "No SAM file specified" unless @sam 
+
+        opts = @write ? "w" : "r"
+        if @binary then  
+          opts += "b" 
+          if @write then
+            unless @compressed then 
+              opts += "u"
+            end
+          end
+        end
+        valid = ["r", "w", "wh", "rb", "wb" , "wbu"]
+        unless valid.include?(opts) then
+          raise SAMException.new(), "Invalid options for samopen: " + opts 
+        end
+
+        samFile = Bio::DB::SAM::Tools.samopen(@sam, opts, nil)
+        if samFile.null? then
+          @sam_file = nil
+          raise SAMException.new(), "File not opened:  " + @sam
+        end
+        @sam_file = Bio::DB::SAM::Tools::SamfileT.new(samFile)
+
+      end
+
+      def to_s()
+        (@binary ? "Binary" : "Text") + " file: " + @sam + " with fasta: " + @fasta_path
+      end
+
+      def close()
+        Bio::DB::SAM::Tools.fai_destroy(@fasta_index) unless @fasta_index.nil? || @fasta_index.null?
+        Bio::DB::SAM::Tools.bam_index_destroy(@sam_index) unless @sam_index.nil? || @sam_index.null?
+        Bio::DB::SAM::Tools.samclose(@sam_file) unless @sam_file.nil? 
+        @sam_file = nil
+        @fasta_index = nil
+      end
+
+      def Sam.finalize(id)
+        id.close()
+        puts "Finalizing #{id}  at #{Time.new}"       
+      end
+
+      def load_index()
+        raise SAMException.new(), "Indexes are only supported by BAM files, please use samtools to convert your SAM file" unless @binary
+        @sam_index = Bio::DB::SAM::Tools.bam_index_load(@sam)
+        if @sam_index.null? then
+          p "Generating index for: " + @sam
+          Bio::DB::SAM::Tools.bam_index_build(@sam)
+          @sam_index = Bio::DB::SAM::Tools.bam_index_load(@sam)
+          raise SAMException.new(), "Unable to generate bam index for: " + @sam if @sam_index.nil? || @sam_index.null?
+        end
+      end
+
+      def load_reference()
+        raise SAMException.new(), "No path for the refernce fasta file. " if @fasta_path.nil?
+
+        @fasta_index = Bio::DB::SAM::Tools.fai_load(@fasta_path)
+
+        if @fasta_index.null? then
+          p "Generating index for: " + @fasta_path
+          Bio::DB::SAM::Tools.fai_build(@fasta_path)
+          @fasta_index =  Bio::DB::SAM::Tools.fai_load(@fasta_path)
+          raise SAMException.new(), "Unable to generate fasta index for: " + @fasta_path if @fasta_index.nil? ||  @fasta_index.null?
+        end
+
+      end
+
+      def average_coverage(chromosome, qstart, len)
+
+        #reference = fetch_reference(chromosome, qstart,len)
+        # len = reference.length if len > reference.length
+
+
+        coverages = chromosome_coverage(chromosome, qstart, len)
+        total = 0
+        len.times{ |i| total= total + coverages[i] }
+        avg_cov = total.to_f / len
+        #LibC.free reference
+        avg_cov
+      end
+
+      def chromosome_coverage(chromosome, qstart, len)
+        #  reference = fetch_reference(chromosome, qstart,len)
+        #  len = reference.length if len > reference.length
+        #p qend.to_s + "-" + qstart.to_s + "framesize " + (qend - qstart).to_s
+        coverages = Array.new(len, 0)
+
+        chr_cov_proc = Proc.new do |alignment|
+          #last = qstart + len
+          #first = qstart
+          #last = alignment.calend if last > alignment.calend
+          #first = alignment.pos if first < alignment.pos
+          # p first
+          last = alignment.calend - qstart
+          first = alignment.pos - qstart
+          if last < first
+            tmp = last
+            last = first
+            first = last
+          end
+
+          #  STDERR.puts "#{first} #{last}\n"
+          first.upto(last-1) { |i|
+
+            coverages[i-1] = 1 + coverages[i-1]  if i-1 < len && i > 0
+          }
+        end
+
+        fetch_with_function(chromosome, qstart, qstart+len,  chr_cov_proc)
+        #p coverages
+        coverages
+      end
+
+      def fetch_reference(chromosome, qstart,qend)
+        load_reference if @fasta_index.nil? || @fasta_index.null? 
+        query = query_string(chromosome, qstart,qend)
+        len = FFI::MemoryPointer.new :int
+        reference = Bio::DB::SAM::Tools.fai_fetch(@fasta_index, query, len)
+        raise SAMException.new(), "Unable to get sequence for reference: "+query if reference.nil?
+
+        reference
+      end
+
+      def query_string(chromosome, qstart,qend)
+        query = chromosome + ":" + qstart.to_s + "-" + qend.to_s 
+        query
+      end
+
+      def fetch(chromosome, qstart, qend)
+        als = Array.new
+        fetchAlignment = Proc.new do |alignment|
+          als.push(alignment.clone)   
+          0  
+        end
+        fetch_with_function(chromosome, qstart, qend, fetchAlignment)
+        als
+      end  
+
+      def fetch_with_function(chromosome, qstart, qend, function)
+        load_index if @sam_index.nil? || @sam_index.null?
+        chr = FFI::MemoryPointer.new :int
+        beg = FFI::MemoryPointer.new :int
+        last = FFI::MemoryPointer.new :int
+        query = query_string(chromosome, qstart,qend)
+        qpointer = FFI::MemoryPointer.from_string(query)
+        header = @sam_file[:header]
+        Bio::DB::SAM::Tools.bam_parse_region(header,qpointer, chr, beg, last) 
+        #raise SAMException.new(), "invalid query: " + query  if(chr.read_int < 0)
+        count = 0;
+
+        fetchAlignment = Proc.new do |bam_alignment, data|
+          alignment =  Alignment.new
+          alignment.set(bam_alignment, header)
+          function.call(alignment)
+          count = count + 1
+          0  
+        end
+        Bio::DB::SAM::Tools.bam_fetch(@sam_file[:x][:bam], @sam_index,chr.read_int,beg.read_int, last.read_int, nil, fetchAlignment)
+        #LibC.free chr
+        #LibC.free beg
+        #LibC.free last
+        #LibC.free qpointer
+        count
+      end
+
+    end
+
+    class Tag
+      attr_accessor :tag, :type, :value
+      def set(str)
+        v = str.split(":")
+        @tag   = v[0]
+        @type  = v[1]
+        @value = v[2]
+      end
+    end
+
+    class Alignment
+
+      def initialize
+        ObjectSpace.define_finalizer(self,
+        self.class.method(:finalize).to_proc)
+      end
+      def Alignment.finalize(object_id)
+
+        #           puts "Object #{object_id} dying at #{Time.new}"
+        #             p  "?" . object_id.al
+        #            p object_id.al
+        LibC.free object_id.al
+        LibC.free object_id.sam
+        LibC.free object_id.calend
+        LibC.free object_id.qlen
+
+        LibC.free object_id.samstr
+      end
+
+      #Attributes from the format
+      attr_accessor :qname, :flag, :rname,:pos,:mapq,:cigar, :mrnm, :mpos, :isize, :seq, :qual, :tags, :al, :samstr
+      #Attributes pulled with the C library
+      attr_accessor  :calend, :qlen
+      #Attrobites frp, the flag field (see chapter 2.2.2 of the sam file documentation)
+      #query_strand and mate_strand are true if they are forward. It is the opposite to the definition in the BAM format for clarity.
+      #primary is the negation of is_negative from the BAM format
+      attr_accessor :is_paired, :is_mapped, :query_unmapped, :mate_unmapped, :query_strand, :mate_strand, :first_in_pair,:second_in_pair, :primary, :failed_quality, :is_duplicate
+
+      def set(bam_alignment, header)
+        #Create the FFI object
+        @al = Bio::DB::SAM::Tools::Bam1T.new(bam_alignment) 
+
+        #set the raw data
+        tmp_str =  Bio::DB::SAM::Tools.bam_format1(header,al)
+        #self.sam =  tmp_str
+        #ObjectSpace.define_finalizer(self, proc {|id| puts "Finalizer one on #{id}" })
+        self.sam = String.new(tmp_str)
+        #LibC.free tmp_str
+        #Set values calculated by libbam
+        core = al[:core]
+        cigar = al[:data][core[:l_qname]]#define bam1_cigar(b) ((uint32_t*)((b)->data + (b)->core.l_qname)) 
+        @calend = Bio::DB::SAM::Tools.bam_calend(core,cigar)
+        @qlen = Bio::DB::SAM::Tools.bam_cigar2qlen(core,cigar)
+
+        #process the flags
+        @is_paired             = @flag & 0x0001 > 0
+        @is_mapped             = @flag & 0x0002 > 0
+        @query_unmapped        = @flag & 0x0004 > 0
+        @mate_unmapped         = @flag & 0x0008 > 0
+        @query_strand          = !(@flag & 0x0010 > 0)
+        @mate_strand           = !(@flag & 0x0020 > 0)
+        @first_in_pair         = @flag & 0x0040 > 0
+        @second_in_pair        = @flag & 0x0080 > 0
+        @primary               = !(@flag & 0x0100 > 0)
+        @failed_quality        = @flag & 0x0200 > 0
+        @is_duplicate          = @flag & 0x0400 > 0
+
+      end
+
+
+      def sam=(sam)
+        #p sam
+        s = sam.split("\t")
+        self.qname = s[0]
+        self.flag  = s[1].to_i
+        self.rname = s[2]
+        self.pos   = s[3].to_i
+        self.mapq  = s[4].to_i
+        self.cigar = s[5]
+        self.mrnm  = s[6]
+        self.mpos  = s[7].to_i
+        self.isize = s[8].to_i
+        self.seq   = s[9]
+        self.qual =  s[10]
+        self.tags = {} 
+        11.upto(s.size-1) {|n| 
+          t = Tag.new 
+          t.set(s[n])
+          tags[t.tag] = t
+        }
+
+
+        #<QNAME> <FLAG> <RNAME> <POS> <MAPQ> <CIGAR> <MRNM> <MPOS> <ISIZE> <SEQ> <QUAL> \
+        #[<TAG>:<VTYPE>:<VALUE> [...]] 
+
+      end
+
+    end
+
+    class SAMException < RuntimeError
+      #we can add further variables to give information of the excpetion
+      def initialize()
+
+      end
+    end
+  end
+end
+
+