bio-ngs 0.3.2.alpha.01

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/Gemfile

@@ -0,0 +1,39 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+gem "bio", ">= 1.4.2"
+gem "bio-bwa", ">= 0.2.2"
+gem "bio-samtools", ">= 0.0.0"
+gem "thor", ">= 0.14.6"
+gem "rubyvis", ">= 0.5.0"
+gem "daemons", ">= 1.1.0"
+gem "ruby-ensembl-api", ">= 1.0.1"
+gem "activerecord",">= 3.0.5"
+gem "sqlite3", ">= 1.3.3"
+gem "bio-blastxmlparser"
+gem "progressbar",">= 0.9.0"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.5.2"
+  gem "rcov", ">= 0"
+  gem "bio", ">= 1.4.2"
+  gem "thor", ">= 0.14.6"
+  gem "ffi", ">= 1.0.6"
+  gem "rubyvis", ">= 0.5.0"
+  gem "rspec", ">= 2.5.0"
+  gem "daemons", ">= 1.1.0"
+  gem "bio-samtools", ">= 0.0.0"
+  gem "ruby-ensembl-api", ">= 1.0.1"
+  gem 'bio-bwa', ">= 0.2.2"
+  gem "activerecord",">= 3.0.5"
+  gem "sqlite3", ">= 1.3.3"
+  gem "bio-blastxmlparser"
+  gem "progressbar",">= 0.9.0"
+  gem "json"
+end

data/Gemfile.lock

@@ -0,0 +1,81 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activemodel (3.1.1)
+      activesupport (= 3.1.1)
+      builder (~> 3.0.0)
+      i18n (~> 0.6)
+    activerecord (3.1.1)
+      activemodel (= 3.1.1)
+      activesupport (= 3.1.1)
+      arel (~> 2.2.1)
+      tzinfo (~> 0.3.29)
+    activesupport (3.1.1)
+      multi_json (~> 1.0)
+    arel (2.2.1)
+    bio (1.4.2)
+    bio-blastxmlparser (1.0.1)
+      bio-logger (>= 1.0.0)
+      nokogiri (>= 1.5.0)
+    bio-bwa (0.2.2)
+    bio-logger (1.0.0)
+      log4r (>= 1.1.9)
+    bio-samtools (0.2.4)
+      bio (>= 1.4.1)
+      ffi
+    builder (3.0.0)
+    daemons (1.1.4)
+    diff-lcs (1.1.3)
+    ffi (1.0.9)
+    git (1.2.5)
+    i18n (0.6.0)
+    jeweler (1.5.2)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    json (1.6.1)
+    log4r (1.1.9)
+    multi_json (1.0.3)
+    nokogiri (1.5.0)
+    progressbar (0.9.1)
+    rake (0.9.2)
+    rcov (0.9.11)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    ruby-ensembl-api (1.0.1)
+      activerecord
+      bio (>= 1)
+    rubyvis (0.5.0)
+    shoulda (2.11.3)
+    sqlite3 (1.3.4)
+    thor (0.14.6)
+    tzinfo (0.3.30)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  activerecord (>= 3.0.5)
+  bio (>= 1.4.2)
+  bio-blastxmlparser
+  bio-bwa (>= 0.2.2)
+  bio-samtools (>= 0.0.0)
+  bundler (~> 1.0.0)
+  daemons (>= 1.1.0)
+  ffi (>= 1.0.6)
+  jeweler (~> 1.5.2)
+  json
+  progressbar (>= 0.9.0)
+  rcov
+  rspec (>= 2.5.0)
+  ruby-ensembl-api (>= 1.0.1)
+  rubyvis (>= 0.5.0)
+  shoulda
+  sqlite3 (>= 1.3.3)
+  thor (>= 0.14.6)

data/LICENSE.txt

@@ -0,0 +1,28 @@
+Copyright (c) 2011 Francesco Strozzi and Raoul J.P. Bonnal
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+
+IMPORTANT NOTE:
+
+The present package includes several binaries that are available under
+different open licenses. Please check the website of each software 
+for further details.

data/README.rdoc

@@ -0,0 +1,240 @@
+= bio-ngs
+
+Provides a framework for handling NGS data with Bioruby.
+
+== What we want to do and support
+* SAMtools
+* BWA
+* Bowtie/TopHat/Cufflinks
+
+* Reporting: text and graphs
+* SGE? 
+
+== Tasks
+We'll try to keep this list updated but just in case type 'biongs -T' to get the most updated list
+NOTE: We are working on these and other tasks, if you find some bugs, please open an issue on Github. 
+=== bwa
+
+* biongs bwa:aln:long [FASTQ] --file-out=FILE_OUT --prefix=PREFIX                                      # Run the aligment for LONG query sequences
+* biongs bwa:aln:short [FASTQ] --file-out=FILE_OUT --prefix=PREFIX                                     # Run the aligment for SHORT query sequences
+* biongs bwa:index:long [FASTA]                                                                        # Make the BWT index for a LONG FASTA database
+* biongs bwa:index:short [FASTA]                                                                       # Make the BWT index for a SHORT FASTA database
+* biongs bwa:sam:paired --fastq=one two three --file-out=FILE_OUT --prefix=PREFIX --sai=one two three  # Convert SAI alignment output into SAM format (paired ends)
+* biongs bwa:sam:single [SAI] --fastq=FASTQ --file-out=FILE_OUT --prefix=PREFIX                        # Convert SAI alignment output into SAM format (single end)
+
+=== convert
+Most of this tasks create sub-processes to speed up conversions
+
+* biongs convert:bam:extract_genes BAM GENES --ensembl-release=N  # Extract GENES from bam. It connects to Ensembl Humnan, release 61
+* biongs convert:bam:sort BAM [PREFIX]
+* biongs convert:bcl:qseq:convert RUN OUTPUT [JOBS]              # Convert a bcl dataset in qseq
+* biongs convert:illumina:fastq:trim_b FASTQ                     # perform a trim on all the sequences on B qualities with Illumina's criteria. Ref to CASAVA manual.
+* biongs convert:qseq:fastq:by_file FIRST OUTPUT                 # Convert a qseq file into fastq
+* biongs convert:qseq:fastq:by_lane LANE OUTPUT                  # Convert all the file in the current and descendant directories belonging to the specified lane in fastq. This command is specific for Illum...
+* biongs convert:qseq:fastq:by_lane_index LANE INDEX OUTPUT      # Convert the qseq from a line and index in a fastq file
+* biongs convert:qseq:fastq:samples_by_lane SAMPLES LANE OUTPUT  # Convert the qseqs for each sample in a specific lane. SAMPLES is an array of index codes separated by commas lane is an integer
+
+=== quality
+
+* biongs quality:boxplot FASTQ_QUALITY_STATS                   # plot reads quality as boxplot
+* biongs quality:fastq_stats FASTQ                             # Reports quality of FASTQ file
+* biongs quality:illumina_b_profile_raw FASTQ --read-length=N  # perform a profile for reads coming fom Illumina 1.5+ and write the report in a txt file
+* biongs quality:illumina_b_profile_svg FASTQ --read-length=N  # perform a profile for reads coming fom Illumina 1.5+
+* biongs quality:reads FASTQ                                   # perform quality check for NGS reads
+* biongs quality:reads_coverage FASTQ_QUALITY_STATS            # plot reads coverage in bases
+* biongs quality:trim FASTQ                                    # trim all the sequences
+
+
+
+=== rna
+
+* biongs rna:idx2fasta INDEX FASTA               # Create a fasta file from an indexed genome, using bowtie-inspect
+* biongs rna:tophat DIST INDEX OUTPUTDIR FASTQS  # run tophat as from command line, default 6 processors
+
+=== root
+
+* biongs project NAME  
+
+=== sff
+
+* biongs sff:extract [FILE]  # Run sff_extract on a SFF file
+
+
+== TasksExamples
+
+   biongs convert:bam:extract_genes your_original.bam BLID,GATA3,PTPRC --ensembl_release=61 --ensembl_specie=homo_sapiens
+
+
+= ForDevelopers
+
+== Wrapper
+Bio-Ngs comes with a build-in wrapper to map binary software directly in BioRuby as objects. From this wrapper object is possible to create Thor task as well, with a lot of sugar.
+=== Wrapping a binary
+
+We want wrap TopHat the famous tool for NGS analyses.
+ * The first step is to include the Wrapping module
+ * set the name of the binary to call. Note: if you avid to set the program name it would not be possible to create a thor task and/or run the program
+ * add the options that the binary accepts, usually if preferred to declare all the options, discover them typing 'your_program_name -h'
+
+    module Bio
+      module Ngs    
+        class Tophat
+          include Bio::Command::Wrapper
+          
+          set_program Bio::Ngs::Utils.binary("tophat/tophat")
+          add_option "output-dir",:type => :string, :aliases => '-o'
+          add_option "min-anchor", :type => :numeric, :aliases => '-a'
+          add_option "splice-mismatches", :type => :numeric, :aliases => '-m'
+          #all other options that you want to expose with the wrapping
+        end #Tophat
+      end #Ngs
+    end #Bio
+
+is possible to use specify in the class
+    use_aliases
+if you want to give a priority to short notation or if your program has only the short notation but you want to extend the task with the long one as well.
+We defined a new property for add_option called
+    :collapse => true
+is used only with use_aliases and it collapse the passed parameter to the short notation - example coming from fastx.rb wrapper, note last row - :
+	module Bio
+	  module Ngs    
+	    module Fastx
+	      class Trim
+	        include Bio::Command::Wrapper
+	        set_program Bio::Ngs::Utils.binary("fastq_quality_trimmer")
+	        use_aliases
+	        add_option :min_size, :type=>:numeric, :default=>20, :aliases => "-l", :desc=>"Minimum length - sequences shorter than this (after trimming)
+	        will be discarded. Default = 0 = no minimum length."
+	        add_option :min_quality, :type=>:numeric, :default=>10, :aliases => "-t", :desc=>"Quality threshold - nucleotides with lower 
+	        quality will be trimmed (from the end of the sequence)."
+	        add_option :output, :type=>:string, :aliases => "-o", :desc => "FASTQ output file.", :collapse=>true
+	        add_option :input, :type=>:string, :aliases => "-i", :desc => "FASTQ input file.", :collapse=>true
+	        add_option :gzip, :type => :boolean, :aliases => "-z", :desc => "Compress output with GZIP."
+	        add_option :verbose, :type => :boolean, :aliases => "-v", :desc => "[-v]         = Verbose - report number of sequences.
+	        If [-o] is specified,  report will be printed to STDOUT.
+	        If [-o] is not specified (and output goes to STDOUT),
+	        report will be printed to STDERR."
+	        add_option :quality_type,  :type=>:numeric, :default => 33, :aliases => "-Q", :desc=>"Quality of fastq file"
+	      end
+	    end
+	  end
+	end
+fastq_quality_trimmer accepts only short notation options and we need to pass an input file, but for some reason popen used internally doesn't work properly with the standard behavior so using :collapse=>true the application will be called:
+   fastq_quality_trimmer -t 20 -t 10 -Q 33 -iinput_file_name.fastq -ooutput_file_name.fastq_trim
+running the program by hand form the command line using a space as separator after -i and  -o works as expected. :collapse is a work around for this problem.
+
+
+
+In case you program work like git which has a main program and the sub_programs for each feature you can use specify the sub program name with
+    set_sub_program "sub_name"
+The wrapper will run the command composing:
+    set_program set_sub_program options arguments
+A practical example of this behavior is samtools which has multiple sub programs view, merge, sort, ....
+SamTools is a particular case because in biongs we are using bio-samtools a binding with FFI and the wrapper because the merge function was too complicated for the binding or at least we do not spent enough time on it, so we make the wrapping for this functionality.
+
+This step is very similar to define a Thor task, add_option is grabbed/inspired from Thor.
+Then you can user this binary also from a bioruby script just calling:
+tophat = Bio::Ngs::Tophat.new
+tophat.params = {"mate-inner-dist"=>dist, "output-dir"=>outputdir, "num-threads"=>1, "solexa1.3-quals"=>true}
+#very important: you can pass parameters that have a name which has been previously declared in the Tophat's class.
+# if you want to pass not declared parameters/options please use arguments.
+tophat.run :arguments=>[index, "#{fastqs}" ]
+
+=== Define the Task
+With our new wrapper, let's define a Thor task on the fly 
+
+    class MyTasks < Thor
+      desc "tophat DIST INDEX OUTPUTDIR FASTQS", "run tophat as from command line, default 6 processors"
+      Bio::Ngs::Tophat.new.thor_task(self, :tophat) do |wrapper, task, dist, index, outputdir, fastqs|
+        wrapper.params = {"mate-inner-dist"=>dist, "output-dir"=>outputdir, "num-threads"=>1, "solexa1.3-quals"=>true}
+        wrapper.run :arguments=>[index, "#{fastqs}" ], :separator=>"="
+        #you tasks here
+      end
+    end
+
+Now is you list the tasks with 'thor -T' you will see the new task.
+
+You can create a new wrapper and configure it and run it from inside a Thor's tasks, like in 'biongs quality:boxplot'
+
+    desc "boxplot FASTQ_QUALITY_STATS", "plot reads quality as boxplot"
+    method_option :title, :type=>:string, :aliases =>"-t", :desc  => "Title (usually the solexa file name) - will be plotted on the graph."
+    method_option :output, :type=>:string, :aliases =>"-o", :desc => "Output file name. default is input file_name with .txt."
+    def boxplot(fastq_quality_stats)
+      output_file = options.output || "#{fastq_quality_stats}.png"
+      boxplot = Bio::Ngs::Fastx::ReadsBoxPlot.new
+      boxplot.params={input:fastq_quality_stats, output:output_file}
+      boxplot.run
+    end
+
+=== Override the run command when the binary dosen't behave normally
+    module Bio
+      module Ngs    
+        module Samtools
+          class View
+            include Bio::Command::Wrapper
+            set_program Bio::Ngs::Utils.binary("samtools")
+            add_option "output", :type => :string, :aliases => '-o'
+
+            alias :original_run :run
+            def run(opts = {:options=>{}, :arguments=>[], :output_file=>nil, :separator=>"="})
+              opts[:arguments].insert(0,"view")
+              opts[:arguments].insert(1,"-b")
+              opts[:arguments].insert(2,"-o")
+              original_run(opts)
+            end
+          end #View
+        end #Samtools
+      end #Ngs
+    end #Bio
+
+=== Notes
+* It's possible to add more sugar and we are working hard on it
+* aliases are not well supported at this time. ToDo
+
+= REQUIREMENTS
+* http://hannonlab.cshl.edu/fastx_toolkit/ (the gem tries to install this tool by itself)
+* http://www.gnuplot.info/
+
+Pleas follow the instruction for your own distribution/operating system
+
+= TODO
+* Report the version of every software installed/used from bio-ngs 
+* Develop fastq quality reports with RibuVis ?
+* Write documentation
+* DONE: Wrapper: better support for aliases and Wrapper#params
+* Convert: re factor code to use ::Daemons
+* DONE:misk_tasks? Extract genes/regions of interest from a bam file and create a smaller bam
+* BRANCH:misk_tasks Explore possibility to user DelayedJobs
+* biongs ann:ensembl:gtf:features:categorize GTF GTF categorize also by chromosome not only by BioType
+* configuration file input,output, experimental design
+* include fastx toolkit, download and compile
+* how to put in background tasks that can be run in parallel
+* is it possible to establish a relation between input data and output data ? like fastq task_selected output/s
+* add description for developers on howto include news external tool with versions.yaml
+ 
+= ChangeLog
+ * 2011.05-26: Bump to version 0.2.0 Complete support for installing fastx and possibly other downloadable tool, inside the gem 
+ * 2011-05-25: Bump to version 0.1.0 Update Cufflinks toolkit 1.0.2. Added initial support to fastx tool kit (binaries not included)
+ * 2011-04-08: Tasks for filtering Ensembl annotation and create classifications. (misk_tasks branch)
+
+
+= Contributing to bio-ngs
+
+Please do not hesitate to contact us:
+
+Raoul J.P. Bonnal, http://github.com/helios, r -at- bioruby -dot- org
+Francesco Strozzi, http://github.com/fstrozzi
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
+* Fork the project
+* Start a feature/bugfix branch
+* Commit and push until you are happy with your contribution
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2011 Francesco Strozzi and Raoul J.P. Bonnal. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,60 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "bio-ngs"
+  gem.homepage = "http://github.com/helios/bioruby-ngs"
+  gem.license = "MIT"
+  gem.summary = %Q{bio-ngs provides a framework for handling NGS data with BioRuby}
+  gem.description = %Q{bio-ngs provides a framework for handling NGS data with BioRuby}
+  gem.email = "francesco.strozzi@gmail.com"
+  gem.authors = ["Francesco Strozzi","Raoul J.P. Bonnal"]
+  # Include your dependencies below. Runtime dependencies are required when using your gem,
+  # and development dependencies are only needed for development (ie running rake tasks, tests, etc)
+  #  gem.add_runtime_dependency 'jabber4r', '> 0.1'
+  #  gem.add_development_dependency 'rspec', '> 1.2.3'
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "bio-ngs #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
+
+
+require 'development_tasks'
+BioNgs::BioNgsTasks.new