transfuse 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 9301f6cf6f1e24b506789da98e79c1b87cc4cfb7
+  data.tar.gz: 066ee9226775d9492a28291d5ee6b16fe79e61e3
+SHA512:
+  metadata.gz: 48eb4011a41c92936752b8e7ec1aa53ab12a2737d62b6c0b733f09edf35aebe17e63562513a8a3a60ce8a0cef17201150d8c8432d649f2f8b498eb2f1b9822d3
+  data.tar.gz: 2339858d5e0e19184cbf9d259506ab7efd5d786fcb5ca95ffa62db13c481efc08d24fea58b47a6637391a1b7a45740074a76961609c2f3164761621657785d25

data/.gitignore

@@ -0,0 +1,3 @@
+# directories
+
+coverage

data/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,87 @@
+PATH
+  remote: .
+  specs:
+    transfuse (0.1.1)
+      bindeps (~> 1.0, >= 1.0.1)
+      bio (~> 1.4, >= 1.4.3)
+      fixwhich (~> 1.0, >= 1.0.2)
+      transrate (= 1.0.0.beta3)
+      trollop (~> 2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ansi (1.5.0)
+    bindeps (1.1.2)
+      fixwhich (~> 1.0, >= 1.0.2)
+    bio (1.4.3.0001)
+    coveralls (0.8.1)
+      json (~> 1.8)
+      rest-client (>= 1.6.8, < 2)
+      simplecov (~> 0.10.0)
+      term-ansicolor (~> 1.3)
+      thor (~> 0.19.1)
+    crb-blast (0.6.4)
+      bindeps (~> 1.0, >= 1.0.3)
+      bio (~> 1.4, >= 1.4.3)
+      fixwhich (~> 1.0, >= 1.0.2)
+      threach (~> 0.2, >= 0.2.0)
+      trollop (~> 2.0)
+    docile (1.1.5)
+    domain_name (0.5.24)
+      unf (>= 0.0.5, < 1.0.0)
+    facade (1.0.6)
+    fix-trinity-output (1.0.0)
+      trollop (~> 2.0)
+    fixwhich (1.0.2)
+      pathname2 (~> 1.4, >= 1.4.4)
+    http-cookie (1.0.2)
+      domain_name (~> 0.5)
+    json (1.8.3)
+    mime-types (2.6.1)
+    minitest (4.7.5)
+    netrc (0.10.3)
+    pathname2 (1.7.3)
+      facade
+    rake (10.4.2)
+    rest-client (1.8.0)
+      http-cookie (>= 1.0.2, < 2.0)
+      mime-types (>= 1.16, < 3.0)
+      netrc (~> 0.7)
+    shoulda-context (1.2.1)
+    simplecov (0.10.0)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    term-ansicolor (1.3.0)
+      tins (~> 1.0)
+    thor (0.19.1)
+    threach (0.2.0)
+    tins (1.5.2)
+    transrate (1.0.0.beta3)
+      bindeps (~> 1.1, >= 1.1.2)
+      bio (~> 1.4, >= 1.4.3)
+      crb-blast (~> 0.5, >= 0.5.0)
+      fix-trinity-output (~> 1.0, >= 1.0)
+      trollop (~> 2.0, >= 2.0.0)
+      yell (~> 2.0, >= 2.0.4)
+    trollop (2.1.1)
+    turn (0.9.7)
+      ansi
+      minitest (~> 4)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.1)
+    yell (2.0.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  coveralls (~> 0.7)
+  rake (~> 10.3, >= 10.3.2)
+  shoulda-context (~> 1.2, >= 1.2.1)
+  simplecov (~> 0.8, >= 0.8.2)
+  transfuse!
+  turn (~> 0.9, >= 0.9.7)

data/README.md

@@ -0,0 +1 @@
+Transfuse

data/Rakefile

@@ -0,0 +1,20 @@
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+end
+
+Rake::TestTask.new do |t|
+  t.name = :corset
+  t.libs << 'test'
+  t.test_files = ['test/test_corset.rb']
+end
+
+Rake::TestTask.new do |t|
+  t.name = :cluster
+  t.libs << 'test'
+  t.test_files = ['test/test_cluster.rb']
+end
+
+desc "Run tests"
+task :default => :test

data/bin/transfuse

@@ -0,0 +1,73 @@
+#!/usr/bin/env ruby
+
+require 'trollop'
+require 'transfuse'
+
+ARGV[0] = "--help" if ARGV.length() == 0
+
+opts = Trollop::options do
+  version Transfuse::VERSION::STRING.dup
+  banner <<-EOS
+
+  Transfuse v#{Transfuse::VERSION::STRING.dup}
+  by Chris Boursnell <cmb211@cam.ac.uk> and
+     Richard Smith-Unna <rds45@cam.ac.uk>
+
+  DESCRIPTION:
+  Merge multiple assemblies.
+
+  USAGE:
+  transfuse <options>
+
+  OPTIONS:
+
+  EOS
+  opt :assembly, "assembly files in FASTA format, comma-separated",
+      :type => String, :required => true
+  opt :scores, "transrate contig score output files, comma-separated",
+      :type => String
+  opt :left, "left reads file in FASTQ format",
+      :type => String
+  opt :right, "right reads file in FASTQ format",
+      :type => String
+  opt :output, "write merged assembly to file",
+      :type => String, :required => :true
+  opt :threads, "number of threads", :type => :int, :default => 1
+  opt :verbose, "be verbose"
+end
+
+transfuse = Transfuse::Transfuse.new opts.threads, opts.verbose
+
+assembly_files = transfuse.check_files opts.assembly
+score_files = transfuse.check_files opts.score if opts.score
+left = transfuse.check_files opts.left if opts.left
+right = transfuse.check_files opts.right if opts.right
+
+if opts.scores
+# load the scores from the comma separated list of files
+  scores = transfuse.load_scores score_files
+elsif opts.left and opts.right
+  scores = transfuse.transrate assembly_files, left, right
+else
+  msg = "Please provide either transrate contig scores as csv files or\n"
+  msg << "left and right fastq files to generate scores using transrate"
+  abort msg
+end
+
+assembly_files = transfuse.filter assembly_files, scores
+
+# concatenate assemblies into one fasta file
+cat = transfuse.concatenate assembly_files
+
+# load fasta sequences from concatenated file into hash
+transfuse.load_fasta cat
+
+# cluster using vsearch or maybe cd-hit-est
+clusters = transfuse.cluster cat
+
+transfuse.sequence_alignment clusters
+# pull out contigs from each cluster based on the scores
+# best = transfuse.select_contigs clusters, scores
+
+# transfuse.output_contigs best, cat, opts.output
+

data/lib/transfuse/cluster.rb

@@ -0,0 +1,113 @@
+module Transfuse
+
+  require 'bio'
+  require 'fixwhich'
+
+  class Cluster
+
+    def initialize threads, verbose
+      @cdhit = Which::which('cd-hit-est').first
+      raise "cd-hit-est was not in the PATH - please install it" unless @cdhit
+      @vsearch = Which::which('vsearch').first
+      raise "vsearch was not in the PATH - please install it" unless @vsearch
+      @id = "1.00"
+      @threads = threads
+      @verbose = verbose
+    end
+
+    def run fasta
+      use_cd_hit = false
+      if use_cd_hit
+        output = cd_hit fasta
+        return parse_output output
+      else
+        output = vsearch fasta
+        return parse_vsearch_output output
+      end
+    end
+
+    def cd_hit fasta
+      puts "running cd-hit-est" if @verbose
+      output = "#{File.basename(fasta, File.extname(fasta))}_cdhit.fa"
+      cdhit_cmd = generate_cdhit_command fasta, output
+      puts cdhit_cmd if @verbose
+      cluster = Cmd.new cdhit_cmd
+      cluster.run output
+      return "#{output}.clstr"
+    end
+
+    def vsearch fasta
+      puts "running vsearch" if @verbose
+      cluster_output = "#{fasta}.clust"
+      vsearch_cmd = generate_vsearch_command fasta, cluster_output
+      cluster = Cmd.new vsearch_cmd
+      cluster.run cluster_output
+      return cluster_output
+    end
+
+    def generate_cdhit_command fasta, out
+      #cd-hit-est -i all.fa  -o cd-hit-clusters.txt -c 0.99999 -T 24 -d 100
+      cmd = "#{@cdhit}"
+      cmd << " -i #{fasta}"
+      cmd << " -o #{out}"
+      cmd << " -c #{@id}" # similarity = number of identical bases /
+                          #              length of shorter sequences
+      cmd << " -T #{@threads}"
+      cmd << " -n 10" # word length - maybe increase??
+      cmd << " -d 100" # output name width
+      cmd << " -g 1" # slower but more accurate mode
+      cmd << " -M 8000" # increase memory
+    end
+
+    def generate_vsearch_command fasta, out
+      vsearch = "#{@vsearch}"
+      vsearch << " --cluster_fast #{fasta}"
+      vsearch << " --id #{@id}"
+      vsearch << " --iddef 0" # cd-hit definition of sequence id
+      vsearch << " --qmask none" # no masking
+      vsearch << " --strand both"
+      vsearch << " --uc #{out}"
+      vsearch << " --threads #{@threads}"
+      return vsearch
+    end
+
+    def parse_output cluster_output
+      puts "parsing cd-hit output #{cluster_output}" if @verbose
+      cluster_id = 0
+      clusters = {}
+      File.open(cluster_output).each_line do |line|
+        if line =~ />Cluster\ ([0-9]+)/
+          cluster_id = $1.to_i
+        elsif line =~ /[0-9]+\s+.+nt,\ >(.+)\.\.\.\sat\s([+\-])\/([0-9\.]+)\%/
+          contig_name = $1
+          strand = $2
+          id = $3.to_f
+          clusters[cluster_id] ||= []
+          clusters[cluster_id] << { :name => contig_name, :strand => strand }
+        elsif line =~ /[0-9]+\s+[0-9]+nt,\s>(.+)\.\.\.\s\*/
+          contig_name = $1
+          strand = "+"
+          clusters[cluster_id] ||= []
+          clusters[cluster_id] << { :name => contig_name, :strand => strand }
+        end
+      end
+      return clusters
+    end
+
+    def parse_vsearch_output cluster_output
+      clusters = {}
+      File.open(cluster_output).each_line do |line|
+        if line.start_with?("S") or line.start_with?("H")
+          cols = line.chomp.split("\t")
+          cluster = cols[1].to_i
+          contig_name = cols[8]
+          clusters[cluster] ||= []
+          clusters[cluster] << contig_name
+        end
+      end
+      return clusters
+    end
+
+  end
+
+end

data/lib/transfuse/cmd.rb

@@ -0,0 +1,38 @@
+require 'open3'
+
+module Transfuse
+
+  class Status
+    def success?
+      return true
+    end
+  end
+
+  class Cmd
+
+    attr_accessor :cmd, :stdout, :stderr, :status
+
+    def initialize cmd
+      @cmd = cmd
+    end
+
+    def run file=nil
+      unless file.nil?
+        if File.exist?(file)
+          @stdout = ""
+          @stderr = ""
+          @status = Status.new
+          return true
+        end
+      end
+      @stdout, @stderr, @status = Open3.capture3 @cmd
+      return false
+    end
+
+    def to_s
+      @cmd
+    end
+
+  end
+
+end

data/lib/transfuse/transfuse.rb

@@ -0,0 +1,201 @@
+class String
+  def revcomp
+    self.tr("ACGT", "TGCA").reverse
+  end
+end
+
+module Transfuse
+
+  require 'csv'
+  require 'transrate'
+
+  class Transfuse
+
+    def initialize threads, verbose
+      @threads = threads
+      @verbose = verbose
+      @clustalo = Which::which('clustalo').first
+      raise "clustalo was not in the PATH - please install it" unless @clustalo
+    end
+
+    def check_files string
+      list = []
+      string.split(",").each do |file|
+        file = File.expand_path(file)
+        if File.exist?(file)
+          puts "#{file} exists" if @verbose
+          list << file
+        else
+          abort "#{file} not found"
+        end
+      end
+      return list
+    end
+
+    def concatenate assemblies
+      catted_fasta = "all-"
+      fasta = []
+      assemblies.each do |name|
+        fasta << File.basename(name, File.extname(name))[0..5]
+      end
+      catted_fasta << fasta.join("-")
+      catted_fasta << ".fa"
+      puts "concatenating assemblies into #{catted_fasta}" if @verbose
+      cmd = "cat "
+      assemblies.each do |file|
+        cmd << " #{file} "
+      end
+      cmd << " > #{catted_fasta}"
+      catter = Cmd.new cmd
+      catter.run catted_fasta
+      return File.expand_path(catted_fasta)
+    end
+
+    def cluster file
+      puts "clustering #{file}" if @verbose
+      cluster = Cluster.new @threads, @verbose
+      return cluster.run file
+    end
+
+    def load_fasta fasta
+      @sequences = {}
+      Bio::FastaFormat.open(fasta).each do |entry|
+        @sequences[entry.entry_id] = entry.seq.to_s
+      end
+    end
+
+    def sequence_alignment clusters
+      clusters.each do |id, list| # threach
+        if list.size > 5
+          seq = ""
+          list.each do |hash|
+            seq << ">#{hash[:name]}\n"
+            if hash[:strand] == "+"
+              seq << "#{@sequences[hash[:name]]}\n"
+            elsif hash[:strand] == "-"
+              seq << "#{@sequences[hash[:name]].revcomp}\n"
+            else
+              abort "Unknown strand #{hash[:strand]}"
+            end
+          end
+          cmd = "echo -e \"#{seq}\" | #{@clustalo} -i - --outfmt fa "
+          cmd << "--output-order tree-order"
+          align = Cmd.new cmd
+          align.run
+          File.open("cluster#{id}.fa", "wb") do |out|
+            out.write align.stdout
+          end
+        end
+      end
+    end
+
+    def load_scores files
+      scores = {}
+      files.each do |file|
+        CSV.foreach(file, :headers => true,
+                          :header_converters => :symbol,
+                          :converters => :all) do |row|
+          name = row[:contig_name]
+          score = row[:score]
+          scores[name] = score
+        end
+      end
+      return scores
+    end
+
+    def filter files, scores
+      filtered_files = []
+      files.each_with_index do |file, index|
+        new_filename = "#{File.basename(file, File.extname(file))}_filtered.fa"
+        unless File.exist?(new_filename)
+          File.open(new_filename, "wb") do |out|
+            puts "opening #{file}..."
+            Bio::FastaFormat.open(file).each do |entry|
+              contig_name = entry.entry_id
+              contig_name = "contig#{index}_#{contig_name}"
+              if scores.key?(contig_name) and scores[contig_name] > 0.01
+                out.write ">#{contig_name}\n"
+                out.write "#{entry.seq}\n"
+              elsif !scores.key?(contig_name)
+                abort "Can't find '#{contig_name}' in scores"
+              end
+            end
+          end
+        end
+        filtered_files << File.expand_path(new_filename)
+      end
+      return filtered_files
+    end
+
+
+    def transrate files, left, right
+      scores = {}
+      scores_file = "scores.csv"
+      if File.exist?(scores_file)
+        puts "loading scores from file" if @verbose
+        File.open(scores_file).each do |line|
+          name, score = line.chomp.split("\t")
+          scores[name] = score.to_f
+        end
+      else
+        files.each_with_index do |fasta, index|
+          puts "transrate on #{fasta}" if @verbose
+          assembly = Transrate::Assembly.new(fasta)
+          transrater = Transrate::Transrater.new(assembly, nil, threads:@threads)
+          transrater.read_metrics(left.join(','), right.join(','))
+          assembly.each do |name, contig|
+            name = "contig#{index}_#{name}"
+            scores[name] = contig.score
+          end
+        end
+        File.open(scores_file, "wb") do |out|
+          scores.each do |name, score|
+            out.write "#{name}\t#{score}\n"
+          end
+        end
+      end
+      return scores
+    end
+
+    def select_contigs clusters, scores
+      puts "selecting contigs" if @verbose
+      best = []
+      clusters.each do |cluster_id, list|
+        best_score = 0
+        best_contig = ""
+        list.each do |contig_name|
+          unless scores[contig_name]
+            abort "can't find #{contig_name} in scores hash\n"
+          end
+          if scores[contig_name] > best_score
+            best_score = scores[contig_name]
+            best_contig = contig_name
+          end
+        end
+        best << best_contig
+      end
+      return best
+    end
+
+    def output_contigs best, fasta, output
+      puts "writing contigs" if @verbose
+      # read in catted fasta sequences
+      sequences = {}
+      Bio::FastaFormat.open(fasta).each do |entry|
+        sequences[entry.entry_id] = entry.seq
+      end
+      File.open(output, "wb") do |out|
+        best.each do |contig_name|
+          if sequences.key?(contig_name)
+            out.write ">#{contig_name}\n"
+            out.write "#{sequences[contig_name]}\n"
+          else
+            puts "can't find #{contig_name} in #{fasta}"
+          end
+        end
+      end
+    end
+
+  end
+
+end

data/lib/transfuse/version.rb

@@ -0,0 +1,17 @@
+module Transfuse
+
+  # Defines the version of this codebase.
+  #
+  # This module is used in help messages and in generating
+  # the Gem. Versions must be incremented in accordance with
+  # Semantic Versioning 2.0 (http://semver.org/).
+  module VERSION
+    MAJOR = 0
+    MINOR = 1
+    PATCH = 1
+    BUILD = nil
+
+    STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')
+  end
+
+end # Transfuse

data/lib/transfuse.rb

@@ -0,0 +1,4 @@
+require 'transfuse/cluster.rb'
+require 'transfuse/cmd.rb'
+require 'transfuse/transfuse.rb'
+require 'transfuse/version.rb'

data/notes.md

@@ -0,0 +1,3 @@
+## Transfuse notes
+
+Musings and mutterings about things we are doing in this project