transrate 1.0.0.beta2 → 1.0.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b90d296192b895be2cb4f5411ccc716264f5f360
-  data.tar.gz: 54fa1d66571912a1526aa7bd0d9ed549a04c8b19
+  metadata.gz: a663790460cbe480bce88af21df3609e8b63a073
+  data.tar.gz: 22b4fb0af30dc78afeee914abd8b93d1e7dd7966
 SHA512:
-  metadata.gz: dc43e7f297ce2d53a03f9a7fcdde87124c605750cb6947d35e32185f5de92f53785fab37302e3abde3e4a2f7736aee3680fb415ab413d914720c025ebbff5f87
-  data.tar.gz: a122304dae2f6009c22e9d396c2b2f874cfa47937e582b1b5e5342cdf2a6d7eaeddcca465b3571022f7979d385f1c4014541a1ac1c5ca241eacaa8e8a4fd84e6
+  metadata.gz: ed8cad8ff40d18fd6ce7b1a78bf8a6dab2ef0b141d98a08c5403e8f766e4b11ab79f96e32f30abdae951c6fee3b5d31c3a2cc1d3c98a5a93c20ea258b6424183
+  data.tar.gz: a60476a414dc3626d80d5d1f8ed967c3dcb79563ce97454b79b5f5595451c4285582650591b5974ac0114980482fadd8f6e6d12c50f392a41cac04fe1562ce86

data/.gitignore

@@ -43,6 +43,8 @@ dryrun
 *.coverage
 
 # c extension build artefacts
+# note that we manually add these in a release,
+# but don't want them updated in general
 Makefile
 transrate.bundle
 transrate.o
@@ -50,3 +52,18 @@ transrate.o
 
 # vagrant stuff
 .vagrant
+
+# large test files
+test/data/sorghum_100.1.fastq
+test/data/sorghum_100.2.fastq
+
+# packaging stuff
+*.tar.gz
+transrate-*
+packaging/bindeps
+packaging/vendor
+packaging/packaging
+traveling-ruby*
+*box
+libruby.so*
+libruby*dylib

data/README.md

@@ -36,7 +36,9 @@ Interested in helping? Great! We particularly would like help with the following
 
 - code review
 - documentation review
-- adding features
+- adding features that are already discussed and approved on the issue tracker
 - tackling bugs
 
 For any of these, please just pick an appropriate issue [on the tracker](https://github.com/Blahah/transrate/issues) and make a pull request.
+
+If you want to suggest, and maybe implement, a new feature, please suggest it on the tracker first. This allows us to give feedback on whether it makes sense given the scope of the software, and for the community to discuss requirements for the feature. *Don't* just implement the feature and make a pull request before dicussing it, because you'll probably find your PR waiting a very long time for review.

data/Rakefile

@@ -1,5 +1,6 @@
 require 'rake/testtask'
 require 'rake/extensiontask'
+require 'bundler/setup'
 
 Rake::ExtensionTask.new('transrate') do |ext|
   ext.lib_dir = "lib/transrate"
@@ -75,6 +76,13 @@ Rake::TestTask.new do |t|
   t.test_files = ['test/test_salmon.rb']
 end
 
+Rake::TestTask.new do |t|
+  t.name = :optimiser
+  t.libs << 'test'
+  t.test_files = ['test/test_optimiser.rb']
+end
+
+
 
 desc "Run tests"
 task :default => :test
@@ -82,22 +90,20 @@ task :default => :test
 # PACKAGING
 
 PACKAGE_NAME = "transrate"
-VERSION = "1.0.0.beta2"
-TRAVELING_RUBY_VERSION = "20141215-2.1.5"
+VERSION = "1.0.0.beta3"
+TRAVELING_RUBY_VERSION = "20150210-2.2.0"
 
 desc "Package your app"
-task :package => ['package:linux:x86_64', 'package:osx']
+task :package => ['package:linux', 'package:osx']
 
 namespace :package do
-  namespace :linux do
-    desc "Package your app for Linux x86_64"
-    task :x86_64 => "packaging/packaging/traveling-ruby-#{TRAVELING_RUBY_VERSION}-linux-x86_64.tar.gz" do
-      create_package("linux-x86_64")
-    end
+  desc "Package your app for Linux x86_64"
+  task :linux => [:bundle_install, "packaging/packaging/traveling-ruby-#{TRAVELING_RUBY_VERSION}-linux-x86_64.tar.gz"] do
+    create_package("linux-x86_64")
   end
 
   desc "Package your app for OS X"
-  task :osx => "packaging/packaging/traveling-ruby-#{TRAVELING_RUBY_VERSION}-osx.tar.gz" do
+  task :osx => [:bundle_install, "packaging/packaging/traveling-ruby-#{TRAVELING_RUBY_VERSION}-osx.tar.gz"] do
     create_package("osx")
   end
 end
@@ -110,19 +116,57 @@ file "packaging/packaging/traveling-ruby-#{TRAVELING_RUBY_VERSION}-osx.tar.gz" d
   download_runtime("osx")
 end
 
+desc "Install gems to local directory"
+task :bundle_install do
+  if RUBY_VERSION !~ /^2\.2\./
+    abort "You can only 'bundle install' using Ruby 2.2, because that's what Traveling Ruby uses."
+  end
+  Bundler.with_clean_env do
+    sh "env BUNDLE_IGNORE_CONFIG=1 bundle install --path packaging/vendor --without development"
+  end
+  sh "rm -f packaging/vendor/*/*/cache/*"
+end
+
 def create_package(target)
-  package_dir = "packaging/#{PACKAGE_NAME}-#{VERSION}-#{target}"
+  package_pref = "#{PACKAGE_NAME}-#{VERSION}-#{target}"
+  package_dir = "packaging/#{package_pref}"
   sh "rm -rf #{package_dir}"
   sh "mkdir -p #{package_dir}/lib/app"
-  sh "cp -r lib #{package_dir}/lib/app/"
-  sh "cp -r bin #{package_dir}/lib/app/"
-  sh "mkdir #{package_dir}/lib/ruby"
-  sh "tar -xzf packaging/packaging/traveling-ruby-#{TRAVELING_RUBY_VERSION}-#{target}.tar.gz -C #{package_dir}/lib/ruby"
+  # copy transrate gem to destination
+  sh "cp -r lib bin deps ext files.txt #{package_dir}/lib/app/"
+  # install travelling ruby
+  sh "mkdir #{package_dir}/lib/app/ruby"
+  sh "tar -xzf packaging/packaging/traveling-ruby-#{TRAVELING_RUBY_VERSION}-#{target}.tar.gz -C #{package_dir}/lib/app/ruby"
+  # install loading script for transrate
   sh "cp packaging/transrate #{package_dir}/transrate"
+  # install bundled gem dependencies
+  sh "cp -pR packaging/vendor #{package_dir}/lib/"
+  sh "cd #{package_dir} && ../minify.sh"
+  sh "cp -r #{package_dir}/lib/vendor/* #{package_dir}/lib/app/"
+  sh "cp Gemfile Gemfile.lock transrate.gemspec #{package_dir}/lib/app/"
+  sh "mkdir #{package_dir}/lib/app/.bundle"
+  sh "cp packaging/bundler-config #{package_dir}/lib/app/.bundle/config"
+  # free up some more space in the package dir
+  sh "rm -rf #{package_dir}/lib/vendor"
+  sh "rm -rf #{package_dir}/lib/app/ruby/*/gems/*/test"
+  # install binary dependencies
+  sh "mkdir -p packaging/bindeps/#{target}"
+  sh "rm -rf packaging/bindeps/#{target}/*"
+  sh "cp test/vagrant/#{target}/*.tar.gz packaging/bindeps/#{target}"
+  sh "mkdir packaging/bindeps/#{target}/{bin,lib}"
+  sh "cd packaging/bindeps/#{target} && " +
+     "find . -maxdepth 1 -name '*.tar.gz' -exec tar -xzf '{}' \\; && " +
+     "mv snap bam-read bin/"
+  sh "cp -r packaging/bindeps/#{target}/{bin,lib} #{package_dir}/"
+  # install c extension
+  sh "cp test/vagrant/#{target}/{transrate,libruby}.* #{package_dir}/lib/"
+  # create package
   if !ENV['DIR_ONLY']
-    sh "tar -czf #{package_dir}.tar.gz #{package_dir}"
+    sh "cd packaging && tar -czf #{package_pref}.tar.gz #{package_pref}"
     sh "rm -rf #{package_dir}"
   end
+  # cleanup
+  sh "rm -rf packaging/vendor packaging/bindeps .bundle"
 end
 
 def download_runtime(target)

data/bin/transrate

@@ -62,25 +62,28 @@ opts = Trollop::options do
   OPTIONS:
 
   EOS
-  opt :assembly, "assembly file(s) in FASTA format, comma-separated",
+  opt :assembly, "Assembly file(s) in FASTA format, comma-separated",
       :type => String
-  opt :reference, "reference proteome file in FASTA format",
+  opt :reference, "Reference proteome file in FASTA format",
       :type => String
-  opt :left, "left reads file in FASTQ format",
+  opt :left, "Left reads file in FASTQ format",
       :type => String
-  opt :right, "right reads file in FASTQ format",
+  opt :right, "Right reads file in FASTQ format",
       :type => String
-  opt :threads, "number of threads to use",
+  opt :threads, "Number of threads to use",
       :default => 8,
       :type => Integer
-  opt :outfile, "prefix filename to use for CSV output",
+  opt :merge_assemblies, "Merge multiple assemblies into file",
+      :type => String
+  opt :outfile, "Prefix filename to use for CSV output",
       :default => 'transrate'
-  opt :loglevel, "the amount of information to print. " +
-                 "one of [error, info, warn, debug]",
+  opt :loglevel, "The amount of information to print. " +
+                 "One of [error, info, warn, debug]",
       :default => 'info'
-  opt :install_deps, "install any missing dependencies"
-  opt :install_read_deps, "install missing dependencies for read metrics only"
-  opt :install_ref_deps, "install missing dependencies for reference metrics only"
+  opt :install_deps, "Install any missing dependencies. One of [all, read, ref]",
+      :type => String, :default => nil
+  # opt :install_read_deps, "install missing dependencies for read metrics only"
+  # opt :install_ref_deps, "install missing dependencies for reference metrics only"
 end
 
 # Check dependencies if they are relevant to the command issued,
@@ -89,7 +92,20 @@ gem_dir = Gem.loaded_specs['transrate'].full_gem_path
 gem_deps = File.join(gem_dir, 'deps', 'deps.yaml')
 blast_dep = File.join(gem_dir, 'deps', 'blast.yaml')
 
-if opts.install_deps || opts.install_read_deps || opts.install_ref_deps
+deps, read_deps, ref_deps = nil
+unless opts.install_deps.nil?
+
+  unless %w[all read red].include? opts.install_deps
+    raise TransrateError.new "install-deps #{opts.install_deps} is not valid. " +
+    "You must specify one of: all, read, ref."
+  end
+
+  deps = opts.install_deps == 'all'
+  read_deps = opts.install_deps == 'read'
+  ref_deps = opts.install_deps == 'ref'
+end
+
+if deps || read_deps || ref_deps
   # user has requested dependency installation
   puts "Checking dependencies"
 
@@ -105,7 +121,9 @@ if opts.install_deps || opts.install_read_deps || opts.install_ref_deps
   end
 
   unless missing.empty?
-    raise TransrateError.new "Failed to install: \n - #{missing.join('\n - ')}"
+    list = missing.collect {|i| "#{i.name}:#{i.version}"}.join("\n - ")
+    msg = "Failed to install: \n - #{list}"
+    raise TransrateError.new msg
   end
 
   puts "All dependencies installed"
@@ -144,7 +162,7 @@ end
 
 # Handle commands
 unless %w[error info warn debug].include? opts.loglevel
-  raise "Loglevel #{opts.loglevel} is not valid. " +
+  raise TransrateError.new "Loglevel #{opts.loglevel} is not valid. " +
   "It must be one of: error, info, warn, debug."
 end
 
@@ -182,14 +200,14 @@ if opts.left and opts.right
   end
 end
 
-def pretty_print_hash hash, width
+def pretty_print_hash hash, width, round=2
   hash.map do |k, v|
     # show as float if there are any decimal places
-    if v.to_f.round(2).to_s.split('.').last.to_i > 0
-      v = v.to_f.round(2)
+    if v.to_f.round(round).to_s.split('.').last.to_i > 0
+      v = v.to_f.round(round)
     end
     if v.is_a? Float
-      v = v.round(2)
+      v = v.round(round)
     end
     pad = (width - (k.to_s.length + v.to_s.length))
     pad = [pad, 0].max
@@ -204,18 +222,35 @@ report_width = 35
 
 # loop through the assemblies, storing their outputs in an array of hashes
 all = []
-opts.assembly.split(',').each do |assembly|
+
+assemblies=opts.assembly
+if opts.merge_assemblies
+  merged_file = opts.merge_assemblies
+  merged = {}
+  assemblies.split(",").each do |file|
+    Bio::FastaFormat.open(file).each do |entry|
+      contig_name = "#{File.basename(file,File.extname(file))}:"
+      contig_name << "#{entry.entry_id}"
+      merged[contig_name] = entry.seq
+    end
+  end
+  logger.info "Merging assemblies into one file...'#{merged_file}'"
+  File.open(merged_file, "wb") do |out|
+    merged.each do |name, seq|
+      out.write ">#{name}\n"
+      out.write "#{seq}\n"
+    end
+  end
+
+  assemblies = merged_file
+end
+
+assemblies.split(',').each do |assembly|
 
   logger.info "Loading assembly: #{assembly}"
 
   a = Assembly.new assembly
-  transrater = Transrater.new(a, r,
-                              left: opts.left,
-                              right: opts.right,
-                              insertsize: opts.insertsize,
-                              insertsd: opts.insertsd,
-                              threads: opts.threads)
-
+  transrater = Transrater.new(a, r, threads: opts.threads)
 
   logger.info "Analysing assembly: #{assembly}"
 
@@ -238,8 +273,7 @@ opts.assembly.split(',').each do |assembly|
   if (opts.left && opts.right)
     logger.info "Calculating read diagnostics..."
     t0 = Time.now
-    read_results = transrater.read_metrics(opts.left,
-                                           opts.right).read_stats
+    read_results = transrater.read_metrics(opts.left, opts.right).read_stats
 
     if read_results
       logger.info "Read mapping metrics:"
@@ -275,9 +309,13 @@ opts.assembly.split(',').each do |assembly|
 
   if (opts.left && opts.right)
     score = transrater.assembly_score
+    optimal, cutoff = transrater.assembly_optimal_score
     unless score.nil?
-      logger.info "TRANSRATE ASSEMBLY SCORE: #{score.round(4)}"
+      pretty_print_hash({:TRANSRATE_ASSEMBLY_SCORE => score}, report_width, 4)
       logger.info "-" * report_width
+      pretty_print_hash({:TRANSRATE_OPTIMAL_SCORE => optimal}, report_width, 4)
+      pretty_print_hash({:TRANSRATE_OPTIMAL_CUTOFF => cutoff}, report_width, 4)
+      pretty_print_hash(transrater.good_contigs, report_width)
     end
   end
 
@@ -309,6 +347,8 @@ opts.assembly.split(',').each do |assembly|
                        .merge(comparative_results)
                        .merge({ :assembly => assembly })
                        .merge({ :score => score })
+                       .merge({ :optimal_score => optimal })
+                       .merge({ :cutoff => cutoff })
 
 end
 

data/deps/deps.yaml

@@ -12,43 +12,35 @@ bam-read:
   binaries:
    - bam-read
   version:
-    number: '1.0.0.beta3'
+    number: '1.0.0.beta4'
     command: 'bam-read'
   url:
     64bit:
-      linux: https://github.com/Blahah/transrate-tools/releases/download/v1.0.0.beta3/bam-read_1.0.0.beta3_linux.tar.gz
-      macosx: https://github.com/Blahah/transrate-tools/releases/download/v1.0.0.beta3/bam-read_1.0.0.beta3_macosx.tar.gz
+      linux: https://github.com/Blahah/transrate-tools/releases/download/v1.0.0.beta4/bam-read_v1.0.0.beta4_linux.tar.gz
+      macosx: https://github.com/Blahah/transrate-tools/releases/download/v1.0.0.beta4/bam-read_v1.0.0.beta4_osx.tar.gz
   unpack: true
 salmon:
   binaries:
     - salmon
   libraries:
-    - libbz2.so.1
-    - libgcc_s.so.1
     - libgomp.so.1
-    - liblzma.so.0
     - libm.so.6
-    - libpthread.so.0
     - librt.so.1
-    - libstdc++.so.6
-    - libtbbmalloc_proxy.so,
-    - libtbbmalloc_proxy.so.2
-    - libtbbmalloc.so
-    - libtbbmalloc.so.2
     - libtbb.so
     - libtbb.so.2
-    - libz.so.1
+    - libtbbmalloc.so
+    - libtbbmalloc.so.2
+    - libtbbmalloc_proxy.so
+    - libtbbmalloc_proxy.so.2
     - libcmph.0.dylib
     - libcmph.dylib
-    - libcmph.la
-    - libstaden-read.la
     - libtbb.dylib
     - libtbbmalloc.dylib
     - libtbbmalloc_proxy.dylib
   version:
-    number: 'Salmon v0.2.7'
-    command: 'salmon --help'
+    number: '0.3'
+    command: 'salmon -v'
   url:
     64bit:
-      linux: https://github.com/kingsfordgroup/sailfish/releases/download/v0.2.7/Salmon-v0.2.7_Ubuntu-12.04.tar.gz
-      macosx: https://github.com/kingsfordgroup/sailfish/releases/download/v0.2.7/Salmon-v0.2.7_MacOSX-10.10.1.tar.gz
+      linux: https://github.com/kingsfordgroup/sailfish/releases/download/v0.3.0/SalmonBeta-v0.3.0_squeeze.tar.gz
+      macosx: https://github.com/kingsfordgroup/sailfish/releases/download/v0.3.0/SalmonBeta-v0.3.0_MacOSX-10.10.2.tar.gz

data/lib/transrate/assembly.rb

@@ -196,17 +196,16 @@ module Transrate
 
     end # basic_bin_stats
 
-    def classify_contigs
+    def classify_contigs cutoff
       # create hash of file handles for each output
       base = File.basename @file
       files = {}
-      %w(good fragmented chimeric bad).each do |type|
+      %w(good bad).each do |type|
         files[type.to_sym] = File.open("#{type}.#{base}", "wb")
       end
       # loop through contigs writing them out to the appropriate file
       @assembly.each_pair do |name, contig|
-        category = contig.classify
-        handle = files[category]
+        handle = files[contig.classify(cutoff)]
         handle.write contig.to_fasta
       end
       # close all the file handles
@@ -215,6 +214,14 @@ module Transrate
       end
     end
 
+    def good_contigs
+      good = 0
+      @assembly.each do |name, contig|
+        good += 1 if contig.classification == :good
+      end
+      good
+    end
+
   end # Assembly
 
 end # Transrate

data/lib/transrate/contig.rb

@@ -12,9 +12,9 @@ module Transrate
     # read-based metrics
     attr_accessor :eff_length, :eff_count, :tpm
     attr_accessor :coverage, :uncovered_bases, :p_uncovered_bases
-    attr_accessor :p_seq_true, :p_unique
+    attr_accessor :p_seq_true
     attr_accessor :low_uniqueness_bases, :in_bridges
-    attr_accessor :p_good, :p_not_segmented, :good
+    attr_accessor :p_good, :p_not_segmented, :good, :classification
     # reference-based metrics
     attr_accessor :has_crb, :reference_coverage
     attr_accessor :hits
@@ -37,11 +37,11 @@ module Transrate
       @p_seq_true = 0
       @uncovered_bases = length
       @p_uncovered_bases = 1
-      @p_unique = 0
       @p_not_segmented = 1
       @score = -1
       @good = 0
       @coverage = 0
+      @classification = :unknown
     end
 
     def each &block
@@ -69,7 +69,6 @@ module Transrate
         :p_bases_covered => p_bases_covered,
         :p_seq_true => p_seq_true,
         :score => score,
-        :p_unique => p_unique,
         :p_not_segmented => p_not_segmented,
         :eff_length => eff_length,
         :eff_count => eff_count,
@@ -249,21 +248,13 @@ module Transrate
     # - fragmented (in_bridges > 0) and no other problems
     # - chimeric (p_not_segmented < 0.25) and no other problems
     # - bad (score < 0.5 and not in any other category)
-    def classify
-      return :good if score >= 0.5
-      # fragmented?
-      if in_bridges > 5
-        if p_not_segmented * p_bases_covered * p_seq_true >= 0.5
-          return :fragmented
-        end
-      end
-      # chimeric?
-      if p_not_segmented < 0.25
-        if p_good * p_bases_covered * p_seq_true >= 0.5
-          return :chimeric
-        end
+    def classify cutoff
+      if score >= cutoff
+        @classification = :good
+      else
+        @classification = :bad
       end
-      return :bad
+      return @classification
     end
 
     def to_fasta