bio-ngs 0.3.2.alpha.01

data/VERSION

@@ -0,0 +1 @@
+0.3.2.alpha.01

data/bin/biongs

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+
+module Bio
+  module Ngs
+
+    path = File.expand_path(File.dirname(__FILE__))
+    $:<<File.join(File.dirname(path),"lib")
+    require 'bio-ngs'
+
+    path = File.expand_path(File.dirname(__FILE__))
+    Dir.glob(File.join(File.dirname(path),"lib","tasks","*.thor")) do |thorfile|
+      Thor::Util.load_thorfile(thorfile)
+    end
+    
+    class Runner < Thor::Runner
+      
+      def help(meth = nil)
+        if meth && !self.respond_to?(meth)
+          initialize_thorfiles(meth)
+          klass, task = Thor::Util.find_class_and_task_by_namespace(meth)
+          klass.start(["-h", task].compact, :shell => self.shell)
+        else
+          puts "\nBio-NGS is an official BioRuby plugin for Next Generation Sequencing data analysis and workflows.\n\n"
+          puts "To view the full list of available tasks and commands type:\n\n"
+          puts "\tbiongs -T\n\n"
+          puts "To get information on specific tasks type:\n\n"
+          puts "\tbiongs help [TASK]\n\n"
+        end
+      end
+
+    end
+    $thor_runner = true
+    Runner.start
+  end
+end

data/bio-ngs.gemspec

@@ -0,0 +1,215 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{bio-ngs}
+  s.version = "0.3.2.alpha.01"
+
+  s.required_rubygems_version = Gem::Requirement.new("> 1.3.1") if s.respond_to? :required_rubygems_version=
+  s.authors = [%q{Francesco Strozzi}, %q{Raoul J.P. Bonnal}]
+  s.date = %q{2011-10-10}
+  s.description = %q{bio-ngs provides a framework for handling NGS data with BioRuby}
+  s.email = %q{francesco.strozzi@gmail.com}
+  s.executables = [%q{biongs}]
+  s.extensions = [%q{ext/mkrf_conf.rb}]
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "bin/biongs",
+    "bio-ngs.gemspec",
+    "ext/mkrf_conf.rb",
+    "lib/bio-ngs.rb",
+    "lib/bio/appl/ngs/bcl2qseq.rb",
+    "lib/bio/appl/ngs/blast.rb",
+    "lib/bio/appl/ngs/bowtie-inspect.rb",
+    "lib/bio/appl/ngs/cufflinks.rb",
+    "lib/bio/appl/ngs/fastx.rb",
+    "lib/bio/appl/ngs/samtools.rb",
+    "lib/bio/appl/ngs/sff_extract.rb",
+    "lib/bio/appl/ngs/tophat.rb",
+    "lib/bio/ngs/converter.rb",
+    "lib/bio/ngs/core_ext.rb",
+    "lib/bio/ngs/db.rb",
+    "lib/bio/ngs/db/migrate/homology/201105030707_create_blastout.rb",
+    "lib/bio/ngs/db/migrate/homology/201105030709_create_goannotation.rb",
+    "lib/bio/ngs/db/migrate/ontology/201105030708_create_go.rb",
+    "lib/bio/ngs/db/migrate/ontology/201105030710_create_gene_go.rb",
+    "lib/bio/ngs/db/migrate/ontology/201105030711_create_gene.rb",
+    "lib/bio/ngs/db/models.rb",
+    "lib/bio/ngs/db/models/homology.rb",
+    "lib/bio/ngs/db/models/ontology.rb",
+    "lib/bio/ngs/ext/bin/common/fastq_coverage_graph.sh",
+    "lib/bio/ngs/ext/bin/common/sff_extract",
+    "lib/bio/ngs/ext/bin/linux/samtools",
+    "lib/bio/ngs/ext/bin/osx/samtools",
+    "lib/bio/ngs/ext/versions.yaml",
+    "lib/bio/ngs/graphics.rb",
+    "lib/bio/ngs/homology.rb",
+    "lib/bio/ngs/ontology.rb",
+    "lib/bio/ngs/quality.rb",
+    "lib/bio/ngs/record.rb",
+    "lib/bio/ngs/task.rb",
+    "lib/bio/ngs/utils.rb",
+    "lib/development_tasks.rb",
+    "lib/enumerable.rb",
+    "lib/tasks/bwa.thor",
+    "lib/tasks/convert.thor",
+    "lib/tasks/history.thor",
+    "lib/tasks/homology.thor",
+    "lib/tasks/ontology.thor",
+    "lib/tasks/project.thor",
+    "lib/tasks/quality.thor",
+    "lib/tasks/rna.thor",
+    "lib/tasks/sff_extract.thor",
+    "lib/templates/README.tt",
+    "lib/templates/db.tt",
+    "lib/wrapper.rb",
+    "spec/converter_qseq_spec.rb",
+    "spec/fixture/s_1_1_1108_qseq.txt",
+    "spec/quality_spec.rb",
+    "spec/sff_extract_spec.rb",
+    "spec/spec_helper.rb",
+    "spec/tophat_spec.rb",
+    "spec/utils_spec.rb",
+    "test/conf/test_db.yml",
+    "test/data/blastoutput.xml",
+    "test/data/gene-GO.json",
+    "test/data/goa_uniprot",
+    "test/data/goslim_goa.obo",
+    "test/helper.rb",
+    "test/test_bio-ngs.rb",
+    "test/test_db.rb",
+    "test/test_homology.rb",
+    "test/test_ngs.rb",
+    "test/test_ontology.rb",
+    "test/test_utils.rb"
+  ]
+  s.homepage = %q{http://github.com/helios/bioruby-ngs}
+  s.licenses = [%q{MIT}]
+  s.require_paths = [%q{lib}]
+  s.rubygems_version = %q{1.8.6}
+  s.summary = %q{bio-ngs provides a framework for handling NGS data with BioRuby}
+  s.test_files = [
+    "spec/converter_qseq_spec.rb",
+    "spec/quality_spec.rb",
+    "spec/sff_extract_spec.rb",
+    "spec/spec_helper.rb",
+    "spec/tophat_spec.rb",
+    "spec/utils_spec.rb",
+    "test/helper.rb",
+    "test/test_bio-ngs.rb",
+    "test/test_db.rb",
+    "test/test_homology.rb",
+    "test/test_ngs.rb",
+    "test/test_ontology.rb",
+    "test/test_utils.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<bio>, [">= 1.4.2"])
+      s.add_runtime_dependency(%q<bio-bwa>, [">= 0.2.2"])
+      s.add_runtime_dependency(%q<bio-samtools>, [">= 0.0.0"])
+      s.add_runtime_dependency(%q<thor>, [">= 0.14.6"])
+      s.add_runtime_dependency(%q<rubyvis>, [">= 0.5.0"])
+      s.add_runtime_dependency(%q<daemons>, [">= 1.1.0"])
+      s.add_runtime_dependency(%q<ruby-ensembl-api>, [">= 1.0.1"])
+      s.add_runtime_dependency(%q<activerecord>, [">= 3.0.5"])
+      s.add_runtime_dependency(%q<sqlite3>, [">= 1.3.3"])
+      s.add_runtime_dependency(%q<bio-blastxmlparser>, [">= 0"])
+      s.add_runtime_dependency(%q<progressbar>, [">= 0.9.0"])
+      s.add_development_dependency(%q<shoulda>, [">= 0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+      s.add_development_dependency(%q<bio>, [">= 1.4.2"])
+      s.add_development_dependency(%q<thor>, [">= 0.14.6"])
+      s.add_development_dependency(%q<ffi>, [">= 1.0.6"])
+      s.add_development_dependency(%q<rubyvis>, [">= 0.5.0"])
+      s.add_development_dependency(%q<rspec>, [">= 2.5.0"])
+      s.add_development_dependency(%q<daemons>, [">= 1.1.0"])
+      s.add_development_dependency(%q<bio-samtools>, [">= 0.0.0"])
+      s.add_development_dependency(%q<ruby-ensembl-api>, [">= 1.0.1"])
+      s.add_development_dependency(%q<bio-bwa>, [">= 0.2.2"])
+      s.add_development_dependency(%q<activerecord>, [">= 3.0.5"])
+      s.add_development_dependency(%q<sqlite3>, [">= 1.3.3"])
+      s.add_development_dependency(%q<bio-blastxmlparser>, [">= 0"])
+      s.add_development_dependency(%q<progressbar>, [">= 0.9.0"])
+      s.add_development_dependency(%q<json>, [">= 0"])
+    else
+      s.add_dependency(%q<bio>, [">= 1.4.2"])
+      s.add_dependency(%q<bio-bwa>, [">= 0.2.2"])
+      s.add_dependency(%q<bio-samtools>, [">= 0.0.0"])
+      s.add_dependency(%q<thor>, [">= 0.14.6"])
+      s.add_dependency(%q<rubyvis>, [">= 0.5.0"])
+      s.add_dependency(%q<daemons>, [">= 1.1.0"])
+      s.add_dependency(%q<ruby-ensembl-api>, [">= 1.0.1"])
+      s.add_dependency(%q<activerecord>, [">= 3.0.5"])
+      s.add_dependency(%q<sqlite3>, [">= 1.3.3"])
+      s.add_dependency(%q<bio-blastxmlparser>, [">= 0"])
+      s.add_dependency(%q<progressbar>, [">= 0.9.0"])
+      s.add_dependency(%q<shoulda>, [">= 0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+      s.add_dependency(%q<bio>, [">= 1.4.2"])
+      s.add_dependency(%q<thor>, [">= 0.14.6"])
+      s.add_dependency(%q<ffi>, [">= 1.0.6"])
+      s.add_dependency(%q<rubyvis>, [">= 0.5.0"])
+      s.add_dependency(%q<rspec>, [">= 2.5.0"])
+      s.add_dependency(%q<daemons>, [">= 1.1.0"])
+      s.add_dependency(%q<bio-samtools>, [">= 0.0.0"])
+      s.add_dependency(%q<ruby-ensembl-api>, [">= 1.0.1"])
+      s.add_dependency(%q<bio-bwa>, [">= 0.2.2"])
+      s.add_dependency(%q<activerecord>, [">= 3.0.5"])
+      s.add_dependency(%q<sqlite3>, [">= 1.3.3"])
+      s.add_dependency(%q<bio-blastxmlparser>, [">= 0"])
+      s.add_dependency(%q<progressbar>, [">= 0.9.0"])
+      s.add_dependency(%q<json>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<bio>, [">= 1.4.2"])
+    s.add_dependency(%q<bio-bwa>, [">= 0.2.2"])
+    s.add_dependency(%q<bio-samtools>, [">= 0.0.0"])
+    s.add_dependency(%q<thor>, [">= 0.14.6"])
+    s.add_dependency(%q<rubyvis>, [">= 0.5.0"])
+    s.add_dependency(%q<daemons>, [">= 1.1.0"])
+    s.add_dependency(%q<ruby-ensembl-api>, [">= 1.0.1"])
+    s.add_dependency(%q<activerecord>, [">= 3.0.5"])
+    s.add_dependency(%q<sqlite3>, [">= 1.3.3"])
+    s.add_dependency(%q<bio-blastxmlparser>, [">= 0"])
+    s.add_dependency(%q<progressbar>, [">= 0.9.0"])
+    s.add_dependency(%q<shoulda>, [">= 0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+    s.add_dependency(%q<bio>, [">= 1.4.2"])
+    s.add_dependency(%q<thor>, [">= 0.14.6"])
+    s.add_dependency(%q<ffi>, [">= 1.0.6"])
+    s.add_dependency(%q<rubyvis>, [">= 0.5.0"])
+    s.add_dependency(%q<rspec>, [">= 2.5.0"])
+    s.add_dependency(%q<daemons>, [">= 1.1.0"])
+    s.add_dependency(%q<bio-samtools>, [">= 0.0.0"])
+    s.add_dependency(%q<ruby-ensembl-api>, [">= 1.0.1"])
+    s.add_dependency(%q<bio-bwa>, [">= 0.2.2"])
+    s.add_dependency(%q<activerecord>, [">= 3.0.5"])
+    s.add_dependency(%q<sqlite3>, [">= 1.3.3"])
+    s.add_dependency(%q<bio-blastxmlparser>, [">= 0"])
+    s.add_dependency(%q<progressbar>, [">= 0.9.0"])
+    s.add_dependency(%q<json>, [">= 0"])
+  end
+end
+

data/ext/mkrf_conf.rb

@@ -0,0 +1,87 @@
+#(c) Copyright 2011 Raoul Bonnal. All Rights Reserved. 
+
+# create Rakefile for shared library compilation
+
+
+path = File.expand_path(File.dirname(__FILE__))
+
+path_external = File.join(path, "../lib/bio/ngs/ext")
+path_lib = File.join(path,"../lib")
+path_binary = File.join(path_external,"bin","common")
+
+
+  File.open(File.join(path,"Rakefile"),"w") do |rakefile|
+    if (ARGV.include?("--biolinux") | ARGV.include?("--no-third-party"))
+      rakefile.write <<-RAKE
+      task :default do
+        puts "Nothing to do, third party software must be, already, installed on the host system. Bye Bye!"
+      end #default
+      RAKE
+    else
+    rakefile.write <<-RAKE
+
+    require 'rbconfig'
+    require 'open-uri'
+    require 'fileutils'
+    include FileUtils::Verbose
+    require 'rake/clean'
+    require 'yaml'
+    require File.join("#{path_lib}","/bio/ngs/utils") #ToDo: I dnk if it is better to require everything or just the utils Bio::Ngs::Utils
+
+    module Gem
+      module Install
+        class << self
+          def foreign_files(path=".")
+            install_files = %w(Rakefile gem_make.out mkrf_conf.rb).map{|fn| File.join(path,fn)}
+            Dir.glob(path+"/*") - install_files
+          end
+        end
+      end
+    end
+
+    versions = YAML.load_file(File.join("#{path_external}","versions.yaml"))
+
+
+    task :download do
+
+      ["common", Bio::Ngs::Utils.os_type].each do |kind_software|
+        #download common libraries or tools
+        #download specific OS binaries or libraries    
+        versions[kind_software].each do |tool, info|
+          filename = "\#{info["basename"]}.\#{info["suffix"]}"
+          Bio::Ngs::Utils.download_with_progress(:url => info["url"], :filename => filename)
+        end
+      end
+    end
+
+    task :compile do
+      ["common", Bio::Ngs::Utils.os_type].each do |kind_software|
+        path_binary = File.join("#{path_external}", 'bin', kind_software)
+        #download common libraries or tools
+        #download specific OS binaries or libraries    
+        versions[kind_software].each do |tool, info|
+          Bio::Ngs::Utils.compile_source(tool, info, "#{path_external}", path_binary) if info["type"]=="source"
+        end #versions
+      end   
+    end #compile
+
+    task :binary do
+      ["common", Bio::Ngs::Utils.os_type].each do |kind_software|
+        path_binary = File.join("#{path_external}", 'bin', kind_software)
+        versions[kind_software].each do |tool, info|
+          Bio::Ngs::Utils.install_binary(tool, info, "#{path_external}", path_binary) if info["type"]=="binary"
+        end #versions
+      end   
+    end #binary
+
+    task :clean do
+      Gem::Install.foreign_files("#{path}").each do |file_to_remove|
+        Dir.exists?(file_to_remove) ? FileUtils.remove_dir(file_to_remove) : FileUtils.rm(file_to_remove)
+      end
+    end #clean
+
+    task :default => [:download,:compile,:binary,:clean]
+
+    RAKE
+   end #unless
+  end  #file

data/lib/bio-ngs.rb

@@ -0,0 +1,54 @@
+#
+#  bio-ngs.rb - Main bio-ngs class
+#
+# Copyright:: Copyright (C) 2011
+#     Raoul Bonnal <r@bioruby.org>,
+#     Francesco Strozzi <francesco.strozzi@gmail.com>
+# License:: The Ruby License
+#
+#
+
+
+
+# External gems
+require 'yaml'
+require 'json'
+require 'daemons'
+require 'bio-samtools'
+require 'thor'
+require 'thor/group'
+require 'thor/runner'
+require 'bio-blastxmlparser'
+require 'bio'
+require 'bio-bwa'
+require 'active_record'
+require 'sqlite3'
+
+# NGS classes
+require 'enumerable'
+require 'wrapper'
+require 'bio/ngs/utils'
+require 'bio/ngs/record'
+require 'bio/ngs/quality'
+require 'bio/ngs/graphics'
+require 'bio/ngs/task'
+require 'bio/ngs/core_ext'
+require 'bio/ngs/converter'
+require 'bio/ngs/db'
+require 'bio/ngs/homology'
+require 'bio/ngs/ontology'
+
+#tools
+require 'bio/appl/ngs/tophat'
+require 'bio/appl/ngs/bowtie-inspect'
+require 'bio/appl/ngs/sff_extract'
+require 'bio/appl/ngs/bcl2qseq' #TODO: FIX THIS BUGGY CODE in THOR TASK
+
+require 'bio/appl/ngs/cufflinks'
+require 'bio/appl/ngs/samtools'
+require 'bio/appl/ngs/fastx'
+require 'bio/appl/ngs/blast'
+
+# history 
+Bio::Ngs::HISTORY_FILE = Dir.pwd+"/.task-history.yml"
+Bio::Ngs::Utils.extend_system_path

data/lib/bio/appl/ngs/bcl2qseq.rb

@@ -0,0 +1,93 @@
+#
+#   bcl2qseq.rb - description
+#
+# Copyright:: Copyright (C) 2011
+#     Raoul Bonnal <@bioruby.org>
+# License:: The Ruby License
+#
+#
+
+
+#require 'bio/command'
+#require 'shellwords'
+#require 'thor'
+#require 'bio/ngs/utils'
+
+# usage: 
+#        setupBclToQseq.py -b /path/to/BaseCalls [--in-place|-o /path/to/Output] [options]
+# 
+# Script to setup the output directory for the conversion of BCLfiles produced by
+# RTA into a standard QSEQ-based base calls folder.
+# 
+# options:
+#   -h, --help            show this help message and exit
+#   -b BASE_CALLS_DIRECTORY, --base-calls-directory=BASE_CALLS_DIRECTORY
+#                         Full path to the Base Calls directory generated by the
+#                         instrument
+#   --in-place            Allow in-place conversion (in the Base Calls
+#                         directory)
+#   -o OUTPUT_DIRECTORY, --output-directory=OUTPUT_DIRECTORY
+#                         Full path to the output directory, if not in --in-
+#                         place mode
+#   -i INTENSITIES_DIRECTORY, --intensities-directory=INTENSITIES_DIRECTORY
+#                         Full path to the directory containing the intensities,
+#                         if different from the parent of the Base Calls
+#                         directory
+#   -f FILTER_DIRECTORY, --filter-directory=FILTER_DIRECTORY
+#                         Full path to the filter directory, if different from
+#                         the Base Calls directory
+#   -p POSITIONS_DIRECTORY, --positions-directory=POSITIONS_DIRECTORY
+#                         Full path to the positions directory, if different
+#                         from the Intensities directory
+#   -P POSITIONS_FORMAT, --positions-format=POSITIONS_FORMAT
+#                         Positions file format (supported formats are:
+#                         {'_pos.txt','.locs','.clocs'})
+#   --no-eamss            Do not apply the EAMSS quality masking
+#   --overwrite           Overwrite the content of the output directory if it
+#                         already exists
+#   --GERALD=GERALD       Path to the GERALD config file
+#   -S, --silent          Do not produce any information or warnings
+#   -Q, --quiet           Produce only warnings. Skips general information
+#   -V, --verbose         Produce additional debug information
+#   -v, --version         Print the version information and exits
+#   -L LOG_FILE, --log-file=LOG_FILE
+#                         Full path to the log file
+#   -c, --include-controls
+#                         Include controls information in Passing Filter column
+#   --ignore-missing-bcl  Assume missing *.bcl files correspond to a missed base
+#                         call (i.e., '.')
+#   --ignore-missing-stats
+#                         Fill in with zeros when *.stats files are missing
+
+
+module Bio
+  module Ngs    
+    class Bclqseq
+
+      include Bio::Command::Wrapper
+
+      set_program Bio::Ngs::Utils.binary("setupBclToQseq.py")
+
+      add_option "base-calls-directory", :type => :string, :aliases =>'-b', :desc => 'Full path to the Base Calls directory generated by the instrument'
+      add_option "in-place",:type => :boolean, :desc => 'Allow in-place conversion (in the Base Calls directory)'
+      add_option "output-directory", :type => :string, :aliases => '-o', :desc => 'Full path to the output directory, if not in --in-place mode'
+      add_option "intensities-directory", :type => :string, :aliases => '-i', :desc => 'Full path to the directory containing the intensities, if different from the parent of the Base Calls directory'
+      add_option "filter-directory", :type => :string , :aliases => '-f', :desc => 'Full path to the filter directory, if different from the Base Calls directory'
+      add_option "positions-directory", :type => :string, :aliases => '-p', :desc => 'Full path to the positions directory, if different from the Intensities directory'
+      add_option "positions-format", :type => :string, :aliases => '-P', :desc => 'Positions file format (supported formats are: {\'_pos.txt\',\'.locs\',\'.clocs\'})'
+      add_option "no-eamss", :type => :boolean, :desc => 'Do not apply the EAMSS quality masking'
+      add_option "overwrite", :type => :boolean, :desc => 'Overwrite the content of the output directory if it already exists'
+      add_option "GERALD", :type => :string, :desc => 'Path to the GERALD config file'
+      add_option "silent", :type => :boolean, :aliases => '-S', :desc => 'Do not produce any information or warnings'
+      add_option "quiet", :type => :boolean, :aliases => '-Q', :desc => 'Produce only warnings. Skips general information'
+      add_option "verbose", :type => :boolean, :aliases => '-V', :desc => 'Produce additional debug information'
+      add_option "log-file", :type => :string, :aliases => '-L', :desc => 'Full path to the log file'
+      add_option "include-controls", :type => :boolean, :aliases => '-c', :desc => 'Include controls information in Passing Filter column'
+      add_option "ignore-missing-bcl", :type => :boolean, :desc => 'Assume missing *.bcl files correspond to a missed base call (i.e., \'.\')'
+      add_option "ignore-missin-stats", :type => :boolean, :desc => 'Fill in with zeros when *.stats files are missing'
+
+    end #Bcl2seq
+  end #Ngs
+end #Bio 
+
+