bio-ngs 0.3.2.alpha.01

data/lib/bio/ngs/quality.rb

@@ -0,0 +1,64 @@
+#
+#  
+#
+# Copyright:: Copyright (C) 2011
+#     Francesco Strozzi <francesco.strozzi@gmail.com>
+#     Raoul J.P. Bonnal <r@bioruby.org>
+# License:: The Ruby License
+#
+#
+
+require 'ostruct'
+
+module Bio
+  module Ngs
+    class FastQuality
+      
+      require 'matrix'
+      
+      attr_accessor :format
+      # as reported in http://dx.doi.org/10.1093/nar/gkp1137
+      # we set the default to fastq_sanger, is a better policy to specify 
+      # ALWAYS the format
+      def initialize(file, format=:fastq_sanger)
+        begin
+          @file = file
+          @stream = Bio::FlatFile.auto(file)  
+          @format = format
+          raise ArgumentError, "the method only accepts FASTQ" unless @stream.dbclass == Bio::Fastq
+        end  
+      end
+      
+      def quality_profile
+        qual = nil
+        tot_reads = 0
+        @stream.each do |read|
+          if qual then
+            qual += Vector[*read.quality_scores]
+          else
+            qual = Vector[*read.quality_scores]
+          end
+          tot_reads += 1
+        end
+        qual = qual/tot_reads.to_f
+        return qual.to_a
+      end
+      
+      # Restart from the beginning of the file and draw a profile of B qalities
+      def track_b_count
+        quals = Hash.new(0) # a new element is initialized at zero
+        reads_count=0
+        @stream = Bio::FlatFile.auto(@file)
+        @stream.each do |read|
+          read.format = format
+          reads_count+=1
+          read_qualities = read.quality_scores
+          read_qualities.each_index do |read_index|            
+            quals[read_index]+=1 if read_qualities[read_index] == 2
+          end #seq
+        end#reads
+        OpenStruct.new(:n_reads=>reads_count, :b_profile=>quals.sort)
+      end
+    end #FastQuality
+  end #Ngs
+end #Bio

data/lib/bio/ngs/record.rb

@@ -0,0 +1,50 @@
+#
+#  
+#
+# Copyright:: Copyright (C) 2011
+#     Francesco Strozzi <francesco.strozzi@gmail.com>
+# License:: The Ruby License
+#
+#
+
+module Bio
+  module Ngs
+    class Record
+      
+      def initialize(file)
+        @filename = file
+        @file = File.new(file,"a+")
+      end
+    
+      def save(name,*args)
+        params = {:name => name, :args => args }
+        unless is_saved?(params) || params[:name] =~/history/
+          @file.write(params.to_yaml)
+          @file.close
+        end
+      end
+      
+      def load   
+          tasks = []
+          YAML.each_document(@file) do |ydoc| 
+            ydoc[:args].flatten!
+            tasks << ydoc
+          end
+          return tasks
+      end
+      
+      def clear
+        history = File.delete(@filename)
+      end
+      
+      private
+      
+      def is_saved?(params)    
+        tasks = []
+        YAML.each_document(@file) {|ydoc| tasks << ydoc}
+        return tasks.include?(params)
+      end
+      
+    end
+  end
+end

data/lib/bio/ngs/task.rb

@@ -0,0 +1,46 @@
+
+#
+#  
+#
+# Copyright:: Copyright (C) 2011
+#     Francesco Strozzi <francesco.strozzi@gmail.com>
+# License:: The Ruby License
+#
+#
+
+
+# opening class Thor::Task to add a save_history method
+class Thor
+  class Task
+
+    def run(instance, args=[])
+      public_method?(instance) ? instance.send(name, *args) : instance.class.handle_no_task_error(name)
+      save_history(instance,args) unless instance.class == Bio::Ngs::Runner or instance.class == Thor::Sandbox::History
+    rescue ArgumentError => e
+      handle_argument_error?(instance, e, caller) ?
+      instance.class.handle_argument_error(self, e) : (raise e)
+    rescue NoMethodError => e
+      handle_no_method_error?(instance, e, caller) ?
+      instance.class.handle_no_task_error(name) : (raise e)
+    end
+    
+    
+    private
+    
+    # process Thor instance and save the task name and parameters
+    def save_history(instance,args)
+      invocation =  instance.instance_variable_get("@_invocations").to_a[0]
+      classes = invocation[0].to_s
+      name = invocation[1].first
+      classes.gsub!(/Thor::Sandbox::/,"")
+      classes.gsub!(/::/,":")
+      classes.downcase!
+      options = [args,instance.options]
+      history = Bio::Ngs::Record.new(Bio::Ngs::HISTORY_FILE)
+      history.save(classes+":"+name,options)
+    end
+    
+    
+  end
+  
+end

data/lib/bio/ngs/utils.rb

@@ -0,0 +1,176 @@
+#
+#  
+#
+# Copyright:: Copyright (C) 2011
+#     Francesco Strozzi <francesco.strozzi@gmail.com>
+#     Raoul J.P. Bonnal <r@bioruby.org>
+# License:: The Ruby License
+#
+#
+require 'find'
+
+module Bio
+  module Ngs
+    class Utils
+      class BinaryNotFound < StandardError
+        def initialize(opts={})
+          @skip_task = opts[:skip_task]
+        end
+
+        def skip_task?
+          @skip_task
+        end
+      end
+      class << self
+        def binary(name)
+          begin
+            if !(plugin_binaries_found = find_binary_files(name)).empty?
+              return plugin_binaries_found.first
+            elsif (os_binary = Bio::Command.query_command ["which", name]) != ""
+              return os_binary.tr("\n","")
+            else
+              raise BinaryNotFound.new(:skip_task=>true), "No binary found with this name: #{name}"
+            end
+          rescue BinaryNotFound => e
+            warn e.message
+          end
+
+        end #binary
+
+        def os_type
+          require 'rbconfig'
+          case Config::CONFIG['host_os']
+          when /darwin/ then return "osx" 
+          when /linux/ then return "linux"
+          when /mswin|mingw/ then raise NotImplementedError, "This plugin does not run on Windows"
+          end
+        end
+
+        # Remove from filename the dot and the extension, adds the tag and the new extension
+        def tag_filename(filename, tag, extension)
+          if filename=~/\..*/
+            filename.gsub(/\..*/, "_#{tag}.#{extension}")
+          else
+            "#{filename}_#{tag}.#{extension}"
+          end
+        end #tag_filename
+
+        def extend_system_path
+          path = File.expand_path(File.dirname(__FILE__))
+          common_dir= File.join(path,"ext","bin","common")
+          os_dir = File.join(path,"ext","bin",self.os_type)
+          sub_dirs = Dir[os_dir+"/*"].select do |file|
+            File.directory?(file)
+          end.map do |dir|
+            ":"+dir
+          end.join
+          ENV["PATH"]+=":"+common_dir+":"+os_dir + sub_dirs
+        end #extend_system_path
+
+        def download_with_progress(opts = {:url => nil, :mode => "", :filename => nil})
+          require "open-uri"
+          require "progressbar"
+          puts "Downloading from #{opts[:url]}"
+          filename = (opts[:filename]) ? opts[:filename] : opts[:url].split('/')[-1]
+          mode = (opts[:mode]) ? opts[:mode] : ""
+          pbar = nil
+          open(opts[:url],"r"+mode,
+          :content_length_proc => lambda {|t|
+            if t && 0 < t
+              pbar = ProgressBar.new('', t)
+              pbar.file_transfer_mode
+            end
+            },
+            :progress_proc => lambda {|s|
+              pbar.set s if pbar
+              }) do |remote|
+                open(filename,"w"+mode) {|file| file.write remote.read(16384) until remote.eof?}
+              end
+              puts "\nDone"     
+            end
+
+            def uncompress_gz_file(file_in)
+              require 'zlib'
+              puts "Uncompressing file #{file_in}"
+              file_out = file_in.gsub(/.gz/,"") 
+              Zlib::GzipReader.open(file_in) {|gz|
+                open(file_out,"w") do |file|
+                  file.write gz.read
+                end
+              }
+              puts "Done\n"          
+            end
+
+
+            def download_and_uncompress(url,fileout)
+              self.download_with_progress(:url => url,:mode => "b",:filename => fileout)
+              self.uncompress_gz_file(fileout)
+            end       
+
+            def uncompress_command(suffix)
+              case suffix
+              when "tar.bz2" then "tar xvfj"
+              when "tar.gz" then "tar xvfz"
+              when "zip" then "unzip"
+              else
+                raise "Unkonw suffix."
+              end
+            end #uncompress_command
+            
+            def uncompress_any(tool_name, tool_record)
+              tool_file_name = "#{tool_record["basename"]}.#{tool_record["suffix"]}"
+              tool_dir_name = tool_record["basename"]
+              uncompress = uncompress_command(tool_record["suffix"])
+              STDERR.puts "#{uncompress} #{tool_file_name}"
+              system "#{uncompress} #{tool_file_name}"
+              STDERR.puts "sto per uscire"
+              if Dir.exists?(tool_dir_name)
+                tool_dir_name
+              elsif Dir.exists?("#{tool_name}-#{tool_record['version']}")
+                "#{tool_name}-#{tool_record['version']}"
+              else
+                raise "BioNGS can not identify the uncompressed destination folder"
+              end
+            end  #uncompress
+            
+            def compile_source(tool_name, tool_record, path_external, path_binary)
+              puts "Uncompressing #{tool_name}..."
+              tool_dir_name = uncompress_any(tool_name, tool_record)
+              puts "Compiling #{tool_name}..."
+              cd(tool_dir_name) do
+                system "PKG_CONFIG_PATH='#{path_external}/bin/common/lib/pkgconfig' ./configure --prefix=#{path_binary} --bindir=#{path_binary}"
+                system "make"
+                system "make install"
+              end #cd
+            end #uncompress_compile
+            
+            def install_binary(tool_name, tool_record, path_external, path_binary)
+              require 'fileutils'
+              include FileUtils::Verbose
+              puts "Uncompressing #{tool_name}"
+              uncompressed_tool_dir_name = uncompress_any(tool_name, tool_record)
+              puts "Installing #{tool_name}"
+              path_binary_tool = File.join(path_binary,tool_name)
+              FileUtils.remove_dir(path_binary_tool) if Dir.exists?(path_binary_tool)
+              FileUtils.mkdir(path_binary_tool) 
+              FileUtils.cp_r "#{uncompressed_tool_dir_name}/.", path_binary_tool, :preserve=>true
+            end #uncompress install binary
+            
+
+            # search in the current gem's directory for installed binaries which the name binary_name
+            # it's a recursive search in common and os specific directories
+            # return an array: empty if the binary can not be found otherwise full path to the binaries
+            # it is up to the user choose which binary to use, it's suggested to use the first in the array
+            # to have a behavirou similar to the search PATH
+            def find_binary_files(binary_name)
+              path = File.expand_path(File.dirname(__FILE__))
+              Find.find(File.join(path,"ext","bin","common"),File.join(path,"ext","bin",self.os_type)).select do |f|
+                File.file?(f) && File.basename(f) == binary_name
+              end
+            end #find_binary_file
+              
+          end #eiginclass
+
+        end # end Utils
+      end # end NGS
+    end # end Bio

data/lib/development_tasks.rb

@@ -0,0 +1,34 @@
+class BioNgs
+  # Rake tasks inspired by Jeweler approach
+  # Include tasks used during gem installation.
+  # Why ? If a developer want's to have a ready to go environment
+  # with bioinformatics software supported by biongs in it's cloned directory.
+  class BioNgsTasks < ::Rake::TaskLib
+    attr_accessor :biongs
+
+    def initialize
+      yield self if block_given?
+
+      define
+    end
+
+    def biongs
+      @biongs ||= self
+    end
+
+    def define
+      namespace :devenv do
+        desc "install external bioinformatics tools, for development, locally -in this directory, cloned from github?-"
+        task :bio_tools do
+          Dir.chdir("ext") do
+            load 'mkrf_conf.rb'
+            `rake -f Rakefile`
+            FileUtils.remove("Rakefile")
+          end
+        end
+      end
+
+      task :devenv => 'devenv:bio_tools'
+    end
+  end
+end

data/lib/enumerable.rb

@@ -0,0 +1,37 @@
+#
+#  enumerable.rb - Add Math functionalities to enumerable
+#
+# ToDo: refactor, it's not the right approach: used in Bio::Ngs::Cufflinks::Diff.process_de
+#
+# Copyright:: Copyright (C) 2011
+#     Raoul Bonnal <r@bioruby.org>,
+#     Francesco Strozzi <francesco.strozzi@gmail.com>
+# License:: The Ruby License
+#
+#
+
+module Enumerable
+
+  #  sum of an array of numbers
+  def sum
+    return self.inject(0){|acc,i|acc +i}
+  end
+
+  #  average of an array of numbers
+  def average
+    return self.sum/self.length.to_f
+  end
+
+  #  variance of an array of numbers
+  def sample_variance
+    avg=self.average
+    sum=self.inject(0){|acc,i|acc +(i-avg)**2}
+    return(1/self.length.to_f*sum)
+  end
+
+  #  standard deviation of an array of numbers
+  def standard_deviation
+    return Math.sqrt(self.sample_variance)
+  end
+
+end  #  module Enumerable

data/lib/tasks/bwa.thor

@@ -0,0 +1,126 @@
+#
+#  
+#
+# Copyright:: Copyright (C) 2011
+#     Francesco Strozzi <francesco.strozzi@gmail.com>
+# License:: The Ruby License
+#
+#
+
+
+class Bwa < Thor
+  
+  class Index < Bwa
+	  
+	  desc "short [FASTA]", "Make the BWT index for a SHORT FASTA database"
+	  method_option :colorspace, :type => :boolean, :desc => "Index in Colorspace" 
+	  method_option :prefix, :type => :string, :desc => "Database index name"
+	  def short(fasta)
+	    real_prefix = (options[:prefix]) ? options[:prefix] : fasta
+  	  Bio::BWA.make_index(:file_in => fasta, :c => options[:colorspace], :prefix => real_prefix)
+	  end
+	
+	  desc "long [FASTA]", "Make the BWT index for a LONG FASTA database"
+	  method_option :colorspace, :type => :boolean, :desc => "Index in Colorspace" 
+	  method_option :prefix, :type => :string, :desc => "Database index name"
+	  def long(fasta)
+      real_prefix = (options[:prefix]) ? options[:prefix] : fasta
+	    Bio::BWA.make_index(:file_in => fasta, :a => "bwtsw",:c => options[:colorspace], :prefix => real_prefix)
+    end
+      
+  end
+  
+  
+  class Aln < Bwa
+    
+    desc "short [FASTQ]", "Run the aligment for SHORT query sequences"
+    method_option :file_out, :type => :string, :desc => "file to write output to instead of stdout", :required => true    
+    method_option :prefix, :type => :string, :desc => "Database prefix", :required => true  
+    method_option :n, :type => :numeric, :desc => "max #diff (int) or missing prob under 0.02 err rate (float) [0.04]"
+    method_option :o, :type => :numeric, :desc => "maximum number or fraction of gap opens [1]"
+    method_option :e, :type => :numeric, :desc => "maximum number of gap extensions, -1 for disabling long gaps [-1]"
+    method_option :i, :type => :numeric, :desc => "do not put an indel within INT bp towards the ends [5]"
+    method_option :d, :type => :numeric, :desc => "maximum occurrences for extending a long deletion [10]"
+    method_option :l, :type => :numeric, :desc => "seed length [32]"
+    method_option :k, :type => :numeric, :desc => "maximum differences in the seed [2]"
+    method_option :m, :type => :numeric, :desc => "maximum entries in the queue [2000000]"
+    method_option :t, :type => :numeric, :desc => "number of threads [1]"
+    method_option :M, :type => :numeric, :desc => "mismatch penalty [3]"
+    method_option :O, :type => :numeric, :desc => "gap open penalty [11]"
+    method_option :E, :type => :numeric, :desc => "gap extension penalty [4]"
+    method_option :R, :type => :numeric, :desc => "stop searching when there are >INT equally best hits [30]"
+    method_option :q, :type => :numeric, :desc => "quality threshold for read trimming down to 35bp [0]"
+    method_option :B, :type => :numeric, :desc => "length of barcode"
+    method_option :c, :type => :boolean, :desc => "input sequences are in the color space"
+    method_option :L, :type => :boolean, :desc => "log-scaled gap penalty for long deletions"
+    method_option :N, :type => :boolean, :desc => "non-iterative mode: search for all n-difference hits"
+    method_option :I, :type => :boolean, :desc => "the input is in the Illumina 1.3+ FASTQ-like format"
+    method_option :b, :type => :boolean, :desc => "the input read file is in the BAM format"
+    method_option :single, :type => :boolean, :desc => "use single-end reads only (effective with -b)"
+    method_option :first, :type => :boolean, :desc => "use the 1st read in a pair (effective with -b)"
+    method_option :second, :type => :boolean, :desc => "use the 2nd read in a pair (effective with -b)"
+    def short(fastq)
+      bwa_options = options.dup
+      bwa_options[:file_in] = fastq
+      Bio::BWA.short_read_alignment(bwa_options.symbolize_keys)
+    end
+    
+    desc "long [FASTQ]", "Run the aligment for LONG query sequences"
+    method_option :file_out, :type => :string, :desc => "file to output results to instead of stdout", :required => true
+    method_option :prefix, :type => :string, :desc => "Database prefix", :required => true    
+    method_option :a, :type => :numeric, :desc => "score for a match [1]"
+    method_option :b, :type => :numeric, :desc => "mismatch penalty [3]"
+    method_option :q, :type => :numeric, :desc => "gap open penalty [5]"
+    method_option :r, :type => :numeric, :desc => "gap extension penalty [2]"
+    method_option :t, :type => :numeric, :desc => "number of threads [1]"
+    method_option :w, :type => :numeric, :desc => "band width [50]"
+    method_option :m, :type => :numeric, :desc => "mask level [0.50]"
+    method_option :T, :type => :numeric, :desc => "score threshold divided by a [30]"
+    method_option :s, :type => :numeric, :desc => "maximum seeding interval size [3]"
+    method_option :z, :type => :numeric, :desc => "Z-best [1]"
+    method_option :N, :type => :numeric, :desc => "seeds to trigger reverse alignment [5]"
+    method_option :c, :type => :numeric, :desc => "coefficient of length-threshold adjustment [5.5]"
+    method_option :H, :type => :boolean, :desc => "in SAM output, use hard clipping rather than soft"
+    def long(fastq)
+      bwa_options = options.dup
+      bwa_options[:file_in] = fastq
+      Bio::BWA.long_read_alignment(bwa_options.symbolize_keys)
+    end
+    
+  end
+
+
+  class Sam < Bwa
+    
+    desc "single [SAI]", "Convert SAI alignment output into SAM format (single end)"
+    method_option :prefix, :type => :string, :required => true, :desc => "Database prefix"
+    method_option :fastq, :type => :string, :required => true, :desc => "FastQ file"
+    method_option :file_out, :type => :string, :required => true, :desc => "File to save the output"
+    method_options :n => :numeric, :r => :string
+    def single(sai)
+      bwa_options = options.dup
+      bwa_options[:sai] = sai
+      Bio::BWA.sai_to_sam_single(bwa_options.symbolize_keys)
+    end
+    
+    desc "paired", "Convert SAI alignment output into SAM format (paired ends)"
+    method_option :prefix, :type => :string, :required => true, :desc => "Database prefix"
+    method_option :file_out, :type => :string, :required => true, :desc => "File to save the output"
+    method_option :sai, :type => :array, :required => true, :desc => "The 2 SAI files"
+    method_option :fastq, :type => :array, :required => true, :desc => "The 2 Fasta/Q files"    
+    method_option :a, :type => :numeric, :desc => "maximum insert size [500]"
+    method_option :o, :type => :numeric, :desc => "maximum occurrences for one end [100000]"
+    method_option :n, :type => :numeric, :desc => "maximum hits to output for paired reads [3]"
+    method_option :N, :type => :numeric, :desc => "maximum hits to output for discordant pairs [10]"
+    method_option :c, :type => :numeric, :desc => "prior of chimeric rate (lower bound) [1.0e-05]"
+    method_option :r, :type => :string, :desc => "read group header line such as `@RG\tID:foo\tSM:bar' [null]"
+    method_option :P, :type => :boolean, :desc => "preload index into memory (for base-space reads only)"
+    method_option :s, :type => :boolean, :desc => "disable Smith-Waterman for the unmapped mate"
+    method_option :A, :type => :boolean, :desc => "disable insert size estimate (force -s)"
+    def paired
+      Bio::BWA.sai_to_sam_paired(options.dup.symbolize_keys)
+    end
+    
+  end
+
+end