bio-ngs 0.3.2.alpha.01

data/lib/tasks/rna.thor

@@ -0,0 +1,126 @@
+require File.expand_path(File.dirname(__FILE__) + '/../bio/ngs/utils')
+require File.expand_path(File.dirname(__FILE__) + '/../wrapper')
+require File.expand_path(File.dirname(__FILE__) + '/../bio/appl/ngs/tophat')
+
+class Rna < Thor
+
+  # you'll end up with 3 accept file, regular, sorted, sorted-indexed
+  desc "tophat DIST INDEX OUTPUTDIR FASTQS", "run tophat as from command line, default 6 processors and then create a sorted bam indexed."
+  method_option :paired, :type => :boolean, :default => false, :desc => 'Are reads paired? If you chose this option pass just the basename of the file without forward/reverse and .fastq'
+  Bio::Ngs::Tophat.new.thor_task(self, :tophat) do |wrapper, task, dist, index, outputdir, fastqs|
+      wrapper.params = task.options #merge passed options to the wrapper.
+      wrapper.params = {"mate-inner-dist"=>dist, "output-dir"=>outputdir, "num-threads"=>6, "solexa1.3-quals"=>true}
+      fastq_files = task.options[:paired] ? ["#{fastqs}_forward.fastq","#{fastqs}_reverse.fastq"]  : ["#{fastqs}"]
+      wrapper.run :arguments=>[index, fastq_files ].flatten, :separator=>"="
+
+      accepted_hits_bam_fn = File.join(outputdir, "accepted_hits.bam")
+      task.invoke "convert:bam:sort", [accepted_hits_bam_fn] # call the sorting procedure.
+  end
+
+  desc "quant GTF OUTPUTDIR BAM ", "Genes and transcripts quantification"
+  Bio::Ngs::Cufflinks::Quantification.new.thor_task(self, :quant) do |wrapper, task, gtf, outputdir, bam|
+    wrapper.params = task.options
+    wrapper.params = {"num-threads" => 6, "output-dir" => outputdir, "GTF" => gtf }
+    wrapper.run :arguments=>[bam], :separator => "="
+  end
+
+  #GTFS_QUANTIFICATION is a comma separated list of gtf file names
+  desc "compare GTF_REF OUTPUTDIR GTFS_QUANTIFICATION", "GTFS_QUANTIFICATIONS, use a comma separated list of gtf"
+  Bio::Ngs::Cufflinks::Compare.new.thor_task(self, :compare) do |wrapper, task, gtf_ref, outputdir, gtfs_quantification|
+    # unless Dir.exists?(outputdir)
+    #   Dir.mkdir(outputdir)
+    # end
+    # Dir.chdir(outputdir)
+    # #I assume GTS_QUANTIFICATION is a comma separated list of single gtf files
+    # gtf_tracking_filename = "#{outputdir}.gtf_tracking"
+    # File.open(gtf_tracking_filename, 'w') do |file|
+    #   file.puts gtfs_quantification.gsub(/,/,"\n")
+    # end #file
+    wrapper.params = task.options
+    wrapper.params = {"outprefix" => outputdir, "gtf_reference"=>gtf_ref}
+    wrapper.run :arguments=>[gtfs_quantification.split(',')]
+    # Dir.chdir("../")
+  end
+  
+  desc "mapquant DIST INDEX OUTPUTDIR FASTQS", "map and quantify"
+  method_option :paired, :type => :boolean, :default => false, :desc => 'Are reads paired? If you chose
+                                                                         this option pass just the basename
+                                                                         of the file without forward/reverse
+                                                                         and .fastq'
+  def mapquant(dist, index, outputdir, fastqs)
+    #tophat
+    invoke :tophat, [dist, index, outputdir, fastqs], :paired=>options.paired
+    #cufflinks quantification on gtf
+    invoke :quant, ["#{index}.gtf", File.join(outputdir,"quantification"), File.join(outputdir,"accepted_hits_sort.bam")]
+  end
+
+#TODO: write test to verify the behaviour
+   desc "idx2fasta INDEX FASTA", "Create a fasta file from an indexed genome, using bowtie-inspect"
+   Bio::Ngs::BowtieInspect.new.thor_task(self, :idx2fasta) do |wrapper, task, index, fasta|
+     puts "Index file... #{index}"
+     puts "Output file... #{fasta}"
+     #Perhaps it would be better that the lib undertands by itself that the second arguments is the output file in case of stdoutput
+     wrapper.run :arguments=>[index], :output_file=>fasta
+   end
+     
+  # desc "idx_fasta [INDEX] [FASTA]", "Create a fasta file from an indexed genome, using bowtie-inspect"
+  # method_option :index, :type => :string, :require => true
+  # method_option :fasta, :type => :string
+  # def idx_to_fasta
+  #   fasta = options.fasta || "#{options.index}.fasta"
+  #   sh "bowtie-inspect #{options.index} > #{fasta}"
+  # end
+  # 
+  # desc "tophat_sr TEXT", "tophat alignment single reads"
+  # method_option :threads, :type=>:numeric, :default=>1
+  # def tophat_sr(text)
+  #   puts self.inspect
+  #   puts options.text
+  #   # TODO tophat --num-threads 1 --solexa1.3-quals --output-dir liver_output Homo_ sapiens/UCSC/hg18/Sequence/BowtieIndex/genome liver.fastq
+  # end
+  # 
+  # desc "tophat_pe", "tophat alignment paired ends reads"
+  # method_option :threads, :type=>:numeric, :default=>1
+  # def tophat_pe
+  #   # TODO 
+  # end
+  # 
+  # desc "assembly_bwt", "assembly using bowtie"
+  # def assembly_bwt
+  # end
+  # 
+  # desc "assembly_bwa", "assembly using bwa"
+  # def assembly_bwa
+  # end
+  # 
+  # desc "cuffscmp", "make a comparison with cufflinkscompare"
+  # def cuffcmp
+  # end
+  # 
+  # desc "cuffsquant", "do a complete quantification usinf cufflinks"
+  # def cuffcmp
+  # end
+  # 
+  # desc "samindex", "index a genome with samtools"
+  # def samindex
+  # end
+  # 
+  # desc "sammerge", "merge two set with samtools"
+  # def sammerge
+  # end
+  # 
+  # desc "qseq_to_fastq_sr [PATH]", "convert a set of qseq files in fastq, single read"
+  # method_option :path, :type => :string, :default => "."
+  # def qseq_to_fastq_sr
+  #   # TODO use code coming from Valeria
+  #   # Bio::Ngs.qseq_to_fastq_si(path)
+  # end
+  # 
+  # desc "qseq_to_fastq_pe [PATH]", "convert a set of qseq files in fastq, paired ends"
+  # method_option :path, :type => :string, :default => "."
+  # def qseq_to_fastq_pe
+  #   # TODO use code coming from Valeria
+  #   # Bio::Ngs.qseq_to_fastq_pe(path)
+  # end
+
+end

data/lib/tasks/sff_extract.thor

@@ -0,0 +1,9 @@
+class Sff < Thor
+  
+  desc "extract [FILE]", "Run sff_extract on a SFF file"
+  Bio::Ngs::SffExtract.new.thor_task(self, :extract) do |wrapper, task, file|
+    wrapper.params = task.options
+    puts wrapper.run :arguments => [file]
+  end
+  
+end

data/lib/templates/README.tt

@@ -0,0 +1,43 @@
+Copyright (c) 2011 Raoul J.P. Bonnal and Francesco Strozzi.
+
+Bio-NGS is an official BioRuby Plugin for Next Generation Sequencing data analysis and workflows.
+
+
+== Description ==
+
+This is an automatically generated directory tree for the '<%= name %>' project.
+
+Here is a tree description:
+
+<%= name %>
+ |
+ |-- data	# the place to store all your raw data and output files
+ |
+ |-- log	# the place to store log and temporary files
+ |
+ |-- scripts	# custom scripts should go here
+ |
+ |-- tasks 	# your custom tasks and workflows for NGS data analyses should go here
+ |
+ |-- conf 	# the place for configuration files
+ |
+ |-- db		# database and migration files
+	
+		
+To view the full list of tasks and analyses available, simply type:
+
+	biongs -T
+
+
+
+== References ==
+
+If you use Bio-NGS for your research please cite the following references:
+
+BioRuby: bioinformatics software for the Ruby programming language. Bioinformatics. 2010 Oct 15;26(20):2617-9. 
+Goto N, Prins P, Nakao M, Bonnal R, Aerts J, Katayama T.
+
+
+
+
+ 

data/lib/templates/db.tt

@@ -0,0 +1,6 @@
+# SQLite version 3.x
+#   gem install sqlite3
+adapter: sqlite3
+database: db/<%= type %>.sqlite3
+pool: 5
+timeout: 5000

data/lib/wrapper.rb

@@ -0,0 +1,225 @@
+#
+#  wrapper.rb - Wrapper class for a generic command
+#
+# Copyright:: Copyright (C) 2011
+#     Raoul Bonnal <r@bioruby.org>
+# License:: The Ruby License
+#
+#
+
+#Notes
+# in case you are developing a new wrapper and want to have a secure environment
+# you must not define the program name and the task will not be cerated.
+
+
+module Bio
+  module Command
+    module Wrapper
+
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      def options
+        self.class.options.merge(@options)
+      end
+
+      def options=(option={})
+        #convert all keys symbols in strings
+        option = option.inject({}){|h,item| h[item[0].to_s]=item[1]; h}
+        @options.merge!(option)
+      end
+
+      def program
+        @program
+      end
+      
+      def sub_program
+        self.class.sub_program
+      end
+
+      def initialize(binary=nil, options={})
+        @program = binary || self.class.program
+        @options = options
+        @params = {}
+      end
+
+      # Parameters are accepted ONLY if the key is present as
+      # a key on the options hash. Sort of validation.
+      # ONLY the valid options are taken into account.
+      # It like a third level of configuration
+      #TODO: check :aliases in options as well, not that now only the main option name is verified
+      def params=(opts={})
+        #add the parameters only if in options
+        opts.each_pair do |parameter, value|
+          @params[parameter.to_s] = value if options.has_key?(parameter.to_s)
+        end        
+      end
+
+      def default_options
+        options.select do |name, opts|
+          opts.has_key? :default
+        end
+      end
+
+      # Return the options, which are by default, and the parameters
+      # setted by the user.
+      # Precedence goes to params setted from user
+      def params
+        default_options.merge(@params)
+      end
+
+      def reset_params
+        @params.clear
+      end
+
+      # Return the options and parameters formmatted as typed in the command line as a string
+      # opts[:separator] is important not all the applications require a "=" for separating options and values
+      # TODO: need to be compliant with Bio::Command ?
+      # TODO: make a test because it should not return an empty string.
+      # TODO: refactor is not beauty
+      def normalize_params(separator="=")
+        #use_aliases?
+        args=params.to_a.map do |option|
+          option_name = option[0].to_s
+          option_values = option[1]
+          #deprecated I'm not sure this code is good (at least the one with kind_of?)
+          if option_values.kind_of? Hash
+            #TODO: refactor this code and verify that the boolean needs a specific options setting.
+            #"--#{option_name}" + ((option_values.has_key?(:type) && option_values[:type]==:boolean) ? ("="+ (option_values[:default] ? "true": "false")) :"=#{option_values[:default]}")
+            if (option_values.has_key?(:type) && option_values[:type]==:boolean && option_values[:default])
+              "--#{option_name}"
+            else
+              use_aliases? && options[option_name].has_key?(:aliases) ? "#{options[option_name][:aliases]} #{option_values[:default]}" : "--#{option_name}#{separator}#{option_values[:default]}"
+            end
+            #deprecated up to here
+          else #is a value of the main hash. (mostly a parameter)
+            if option_values == true
+              use_aliases? && options[option_name].has_key?(:aliases) ? options[option_name][:aliases] : "--#{option_name}"
+            elsif option_values != false
+              use_aliases? && options[option_name].has_key?(:aliases) ? "#{options[option_name][:aliases]}#{options[option_name][:collapse] ? "": " "}#{option_values}" : "--#{option_name}#{separator}#{option_values}"
+            end
+          end
+        end
+      end
+
+      def output
+        self.class.output || :file
+      end
+
+      def use_aliases?
+        self.class.aliases
+      end
+
+
+      # If parameters are passed they will overwrite those already defined
+      # but will not save the changes
+      # opts = {:options=>{}, :arguments=>[]}
+      # in the particular case the user wants to submit other options
+      # these must be passed in arguments like {"option_name"=>value} similar when settin params
+      # opts[:separator] is important not all the applications require a "=" for separating options and values
+      # TODO handle output file with program which writes on stdout
+      #TODO: refactor mostly due to stdin/out
+      def run(opts = {:options=>{}, :arguments=>[], :output_file=>nil, :separator=>"="})
+        if program.nil?
+          warn "WARNING: no program is associated with #{class_name.upcase} task."
+          return nil
+        end  
+        #REMOVE        params = opts[:options]
+        if output == :stdout 
+          raise "Can't write to any output file. With a program which writes on stdout you must provide a file name" if opts[:output_file].nil?
+          file_stdlog = File.open(opts[:output_file], 'w')
+          file_errlog = File.open(opts[:output_file]+".err",'w')
+
+          Bio::Command.call_command_open3([program, sub_program, normalize_params(opts[:separator]), opts[:arguments]].flatten.compact) do |pin, pout, perr|
+            pout.sync = true
+            perr.sync = true           
+            t = Thread.start {pout.lines{|line| file_stdlog.puts line}}
+            begin
+              pin.close
+            ensure
+              t.join
+            end
+          end #command call open3
+          file_stdlog.close
+          file_errlog.close
+        else
+          # puts "Normlized #{normalize_params(opts[:separator])}"
+          # puts "Arguments #{opts[:arguments]}"
+#puts [program, sub_program, normalize_params(opts[:separator]), opts[:arguments]].flatten.compact.inspect
+
+        Bio::Command.query_command([program, sub_program, normalize_params(opts[:separator]), opts[:arguments]].flatten.compact)
+        end #if
+      end #run
+
+      # Inject into the Thor::Sandbox::TaskName (klass) the options defined for this 
+      # wrapper
+      # Example of call
+      #   desc "task_name ARG_ONE ARG_SECOND", "run tophat as from command line"
+      #   Bio::Ngs::Tophat.new.thor_task(self, :tophat) do |wrapper, task, ARG_ONE ARG_SECOND|
+      #       puts ARG_ONE
+      #       puts ARG_SECOND
+      #       #you tasks here
+      #   end      
+      def thor_task(klass, task_name, &block)
+        if program.nil?
+          warn "WARNING: no program is associated with #{class_name.upcase} task, does not make sense to create a thor task."
+          return nil
+        end          
+        if klass
+          wrapper = self   
+          klass.class_eval do            
+            wrapper.options.each_pair do |name, opt|
+              method_option name, opt
+            end #each_pair
+            # Thor's behavior should be respected passing attributes
+            define_method task_name do |*args|
+              #it's mandatory that the first and second parameter are respectively wrapper and task
+              raise ArgumentError, "wrong number of arguments (#{args.size} for #{block.parameters.size-2})" if args.size != block.parameters.size-2
+              yield wrapper, self, *args
+            end
+          end#class_eval
+        end #klass
+      end #thor_task
+
+      #Return the class name
+      def class_name
+        self.class.name.split("::").last.downcase
+      end
+
+      module ClassMethods
+        #TODO: do I need to set a default program name using class name or not ?
+        #       or do we need to specify somewhere a defaitl path and looking for a real binary ?
+
+        OUTPUT = [:file, :stdout, :stdin]
+
+        # output = {:file=>true, :stdout=>}
+        attr_accessor :output, :program, :options, :aliases, :sub_program
+
+        #TODO I don't like this way, Is it possible to configure the variable as global and default ?
+        def set_output(output_type=:file)
+          if OUTPUT.include? output_type
+            @output = output_type
+          else
+            raise "Output type #{output_type} is not suported. Valid types are #{OUTPUT}"
+          end
+        end
+
+        def use_aliases
+          @aliases = true
+        end
+
+        # external_parameter can be an array a string or an hash
+        # def validate_parameters(external_parameters)
+        def add_option(name, opt={})
+          @options = (@options || {}).merge(name.to_s=>opt)
+        end
+
+        alias set_program program=
+        alias set_sub_program sub_program=
+
+      end #ClassMethods
+
+    end #Wrapper
+  end #Command
+end #Bio

data/spec/converter_qseq_spec.rb

@@ -0,0 +1,56 @@
+#
+#  converter_qseq_spec.rb - RSpec Test
+#
+# Copyright:: Copyright (C) 2011
+#     Raoul Bonnal <r@bioruby.org>
+# License:: The Ruby License
+#
+#
+
+
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "Converter" do 
+  describe "Qseq" do
+    it "convert two qseq lines into fastq" do
+      qseq = Bio::Ngs::Converter::Qseq.new(:pe)
+      qseq.buffer = "H125    98      1       1108    1586    1989    CGATGT  1       CAGA.C.................A.....GAATGGCATGGATCAAGAAAATCCCCCTTGTGAAGAAGAATCAGCAG    BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB    0\nH125    98      1       1108    1188    2036    CGATGT  1       CTTGTATGCAGCATCCCCTTCTTGCCTAGGGACTTGAAGGGCCAGGCTTCCTGTCATTGCCTCACTCAAATGTAGC    gggggggggggggegggggffggeggegggeagge^ggdbcgggcdgedegfggffff^ffffefdeeZefccceg    1"
+      fastqs = []
+      qseq.to_fastq do |fastq|
+        fastqs << fastq if fastq
+      end
+      fastqs.first.should == "@H125:1:1108:1188:2036#0/1\nCTTGTATGCAGCATCCCCTTCTTGCCTAGGGACTTGAAGGGCCAGGCTTCCTGTCATTGCCTCACTCAAATGTAGC\n+\ngggggggggggggegggggffggeggegggeagge^ggdbcgggcdgedegfggffff^ffffefdeeZefccceg"
+    end
+    
+    it "convert a qseq File into fastq for parierd ends" do
+      qseq = Bio::Ngs::Converter::Qseq.new(:pe)
+      buffer_filename = File.dirname(__FILE__) + "/fixture/s_1_1_1108_qseq.txt"
+      fastq_filename  = File.dirname(__FILE__) + "/fixture/s_1_1_1108_qseq.fastq"
+      qseq.buffer = File.open(buffer_filename,'r')
+      fastq_file = File.open(fastq_filename, 'w')
+      qseq.to_fastq do |fastq|
+        fastq_file.puts fastq if fastq
+      end
+      fastq_file.close
+      fastq_file = File.open(fastq_filename, 'r')
+      fastq_file.readlines[0..4].join("").should == "@H125:1:1108:1188:2036#0/1\nCTTGTATGCAGCATCCCCTTCTTGCCTAGGGACTTGAAGGGCCAGGCTTCCTGTCATTGCCTCACTCAAATGTAGC\n+\ngggggggggggggegggggffggeggegggeagge^ggdbcgggcdgedegfggffff^ffffefdeeZefccceg\n"
+      fastq_file.close
+      #File.delete(fastq_filename)
+    end
+    
+    it "get statistics from converted reads" do
+      qseq = Bio::Ngs::Converter::Qseq.new(:pe)
+      buffer_filename = File.dirname(__FILE__) + "/fixture/s_1_1_1108_qseq.txt"
+      fastq_filename  = File.dirname(__FILE__) + "/fixture/s_1_1_1108_qseq.fastq"
+      qseq.buffer = File.open(buffer_filename,'r')
+      fastq_file = File.open(fastq_filename, 'w')
+      qseq.to_fastq do |fastq|
+        fastq_file.puts fastq if fastq
+      end
+      fastq_file.close
+      #File.delete(fastq_filename)      
+      qseq.stats.should == {:reads_total=>100, :reads_passed=>1, :reads_rejected=>99, :bases_passed_total=>76, :bases_rejected_total=>7524, :bases_passed_with_b_quality=>0, :bases_rejected_with_b_quality=>4004, :bases_passed_with_n=>0, :bases_rejected_with_n=>12}
+    end    
+    
+  end
+end