finishm 0.0.1

data/lib/assembly/hybrid_velvet_graph.rb

@@ -0,0 +1,70 @@
+# A velvet Graph where the nodes and arcs are in Ruby, but the NodedReads are in C
+class Bio::FinishM::HybridGraph < Bio::Velvet::Graph
+  include Bio::FinishM::Logging
+
+  attr_accessor :bio_velvet_underground_graph
+
+  def initialize(bio_velvet_graph, bio_velvet_underground_graph)
+    @bio_velvet_underground_graph = bio_velvet_underground_graph
+
+    @nodes = NodeArray.new(bio_velvet_graph, bio_velvet_underground_graph, self)
+    @hash_length = bio_velvet_graph.hash_length
+    @number_of_nodes = bio_velvet_graph.number_of_nodes
+    @number_of_sequences = bio_velvet_graph.number_of_sequences
+    @arcs = bio_velvet_graph.arcs
+  end
+
+  class NodeArray
+    include Enumerable
+
+    def initialize(bio_velvet_graph, bio_velvet_underground_graph, parent_graph)
+      @bio_velvet_graph = bio_velvet_graph
+      @bio_velvet_underground_graph = bio_velvet_underground_graph
+      @parent_graph = parent_graph
+    end
+
+    def []=(node_id, value)
+      raise "method not implemented"
+    end
+
+    def [](node_id)
+      bio_velvet_node = @bio_velvet_graph.nodes[node_id]
+      bio_velvet_node.short_reads = LazyShortReadArray.new(@bio_velvet_underground_graph.nodes[node_id])
+      bio_velvet_node.parent_graph = @parent_graph
+      return bio_velvet_node
+    end
+
+    def delete(node)
+      @bio_velvet_graph.nodes.delete node
+    end
+
+    def length
+      @bio_velvet_graph.nodes.length
+    end
+
+    def each(&block)
+      @bio_velvet_graph.nodes.each do |node|
+        block.yield self[node.node_id]
+      end
+    end
+  end
+
+  # Hold pointers to short reads, but don't actually create them until required.
+  # Utilizes method_missing to pick up calls made to the underlying object
+  class LazyShortReadArray
+    @short_reads = nil
+
+    def method_missing(m, *args, &block)
+      read_short_reads.send(m, *args, &block)
+    end
+
+    def read_short_reads
+      @short_reads ||= @parent_node.short_reads
+    end
+
+    def initialize(node)
+      @parent_node = node
+    end
+  end
+end
+

data/lib/assembly/input_genome.rb

@@ -0,0 +1,182 @@
+class Bio::FinishM::InputGenome
+  attr_accessor :scaffolds, :filename, :numbered_probes
+  include Bio::FinishM::Logging
+
+  # Return an array of parsed fasta files
+  def self.parse_genome_fasta_files(fasta_files, hangover_length, options = {})
+    genomes = []
+    current_probe_number = 1
+    fasta_files.each do |genome_fasta|
+      genome = Bio::FinishM::InputGenome.new(
+        genome_fasta, hangover_length, :starting_probe_number => current_probe_number
+        )
+      current_probe_number += genome.number_of_probes
+
+      genomes.push genome
+    end
+    return genomes
+  end
+
+  # Given a fasta file, setup a genome for wandering or gapfilling.
+  #
+  # Options:
+  # :starting_probe_number: number probes starting from this number (default 1)
+  def initialize(genome_fasta, hangover_length, options = {})
+    starting_probe_number = options[:starting_probe_number]
+    starting_probe_number ||= 1
+
+    @filename = genome_fasta
+    scaffolds = Bio::FinishM::ScaffoldBreaker.new.break_scaffolds(genome_fasta)
+    @scaffolds = remove_overly_short_contigs!(scaffolds, hangover_length)
+
+    # Remove scaffolds that have no good
+    num_too_short_scaffolds = 0
+    @scaffolds.reject! do |scaff|
+      rej = scaff.contigs.empty?
+      num_too_short_scaffolds += 1 if rej
+      rej
+    end
+    if num_too_short_scaffolds > 0
+      log.warn "Removed #{num_too_short_scaffolds} scaffolds entirely as they were too short (or made up of all short contigs)"
+    end
+
+    generate_numbered_probes(hangover_length, starting_probe_number)
+  end
+
+  def remove_overly_short_contigs!(scaffolds, hangover_length)
+    num_contigs_removed = 0
+    scaffolds.each do |scaffold|
+      num_original_contigs = scaffold.contigs.length
+      scaffold.contigs.reject! do |contig|
+        contig.length < 2*hangover_length
+      end
+      num_contigs_removed += num_original_contigs - scaffold.contigs.length
+    end
+    if num_contigs_removed > 0
+      log.warn "Removed #{num_contigs_removed} contigs from within scaffolds as they were too short"
+    end
+    return scaffolds
+  end
+
+  def generate_numbered_probes(overhang, starting_probe_number)
+    @numbered_probes = []
+    @probe_number_to_scaffold_and_contig_and_side = {}
+
+    current_probe_number = starting_probe_number
+    overly_short_sequence_count = 0
+    @scaffolds.each_with_index do |scaffold, scaffold_index|
+      scaffold.contigs.each_with_index do |contig, contig_index|
+        if contig.sequence.length < 2*overhang
+          log.warn "Not attempting to make connections from overly short contig: it is the #{contig_index+1}th contig in scaffold `#{scaffold.name}' from the genome in `#{@filename}')"
+          overly_short_sequence_count += 1
+          nil
+        else
+          sequence = contig.sequence
+
+          probe1 = NumberedProbe.new
+          probe1.contig = contig
+          probe1.number = current_probe_number; current_probe_number += 1
+          probe1.side = :start
+          fwd2 = Bio::Sequence::NA.new(sequence[0...overhang])
+          probe1.sequence = fwd2.reverse_complement.to_s
+
+          probe2 = NumberedProbe.new
+          probe2.contig = contig
+          probe2.number = current_probe_number; current_probe_number += 1
+          probe2.side = :end
+          probe2.sequence = sequence[(sequence.length-overhang)...sequence.length].to_s
+
+          @numbered_probes[scaffold_index] ||= []
+          @numbered_probes[scaffold_index][contig_index] = [probe1, probe2]
+
+          @probe_number_to_scaffold_and_contig_and_side[probe1.number] = [scaffold, contig, :start]
+          @probe_number_to_scaffold_and_contig_and_side[probe2.number] = [scaffold, contig, :end]
+        end
+      end
+    end
+    log.debug "Generated #{current_probe_number-starting_probe_number} probes for #{@filename}" if log.debug?
+    if overly_short_sequence_count > 0
+      log.warn "Skipping #{overly_short_sequence_count} overly short contigs" if log.warn?
+    end
+  end
+
+  def number_of_probes
+    @numbered_probes.flatten.length
+  end
+
+  def each_numbered_probe
+    @numbered_probes.flatten.each do |probe|
+      yield probe
+    end
+  end
+
+  def each_gap_probe_pair(scaffold_index)
+    last_probe_pair = nil
+    @numbered_probes[scaffold_index].each do |probe_pair|
+      unless probe_pair.nil?
+        unless last_probe_pair.nil?
+          yield last_probe_pair[1], probe_pair[0]
+        end
+        last_probe_pair = probe_pair
+      end
+    end
+  end
+
+  def each_scaffold_end_numbered_probe
+    @numbered_probes.each_with_index do |scaffold_indices, i|
+      # yield the first and last probe of this scaffold
+      yield scaffold_indices[0][0]
+      yield scaffold_indices[-1][1]
+    end
+  end
+
+  def first_probe(scaffold_index)
+    @numbered_probes[scaffold_index][0][0]
+  end
+
+  def last_probe(scaffold_index)
+    @numbered_probes[scaffold_index][-1][1]
+  end
+
+  # Return true if probe number given is the probe at the beginning of the scaffold
+  # or false if it is at the end. raise if unknown.
+  def probe_at_start_of_scaffold?(probe_index)
+    scaffold, contig, side = @probe_number_to_scaffold_and_contig_and_side[probe_index]
+    if side == :start
+      return true
+    elsif side == :end
+      return false
+    else
+      raise
+    end
+  end
+
+  def probe_sequences
+    seqs = []
+    each_numbered_probe do |probe|
+      unless probe.nil?
+        seqs.push probe.sequence
+      end
+    end
+    return seqs
+  end
+
+  # The length of the gap between contig i and contig i+1 from the specified scaffold
+  # (both numbers are 0-based indices)
+  def gap_length(scaffold_index, gap_index)
+    @scaffolds[scaffold_index].gaps[gap_index].length
+  end
+
+  def human_scaffold_end_name(probe_index)
+    scaffold, contig, side = @probe_number_to_scaffold_and_contig_and_side[probe_index]
+    "scaffold '#{scaffold.name}' contig '#{genome.contig_number(contig) }' #{side.to_s}"
+  end
+end
+
+class NumberedProbe
+  attr_accessor :number, :contig, :side, :sequence
+
+  def index
+    @number - 1
+  end
+end

data/lib/assembly/kmer_coverage_based_path_filter.rb

@@ -0,0 +1,65 @@
+require 'bio'
+
+module Bio::AssemblyGraphAlgorithms
+  class KmerCoverageBasedPathFilter
+    include Bio::FinishM::Logging
+
+    # Remove all paths where the kmer coverage is below the threshold at
+    # any point along the path.
+    #
+    # :paths: an iterable collection of paths
+    # :kmer_hash: KmerMultipleAbundanceHash
+    # :thresholds: minimum coverages (min numbers of full kmers) required at each point along the path
+    #
+    # Optional options
+    # * :exclude_ending_length => don't filter out paths based on kmers close to the end e.g. :exclude_ending_length => 2 means the first 2 and last 2 kmers are not considered
+    def filter(paths, kmer_hash, thresholds, options={})
+      # sanity check
+      unless kmer_hash.number_of_abundances == thresholds.length
+        raise "Unexpectedly found a different number of thresholds and kmer abundance columns"
+      end
+
+      passable_paths = []
+      paths.each do |path|
+        passable = true
+        seq = path.sequence
+
+        # remove ends of sequence if kmers don't count at the ends
+        if options[:exclude_ending_length]
+          ex = options[:exclude_ending_length]
+          seq = seq[ex...(seq.length-ex)]
+        end
+
+        Bio::Sequence::NA.new(seq).window_search(kmer_hash.kmer_length,1) do |kmer|
+          kmer_hash[kmer].each_with_index do |abundance, i|
+            if abundance < thresholds[i]
+              passable = false
+              log.debug "Failing trail #{path.sequence} due to insufficent abundance (#{abundance} from #{kmer_hash[kmer]}) for #{kmer}" if log.debug?
+              break
+            end
+          end
+          break if !passable
+        end
+        passable_paths.push path if passable
+      end
+      return passable_paths
+    end
+
+    # Write coverages to the given IO object as a tab-separated file similar
+    # to the output of "samtools depth"
+    def write_depths(io, trail_name, trail_sequence, kmer_hash)
+      #write header
+      io.print %w(trail position).join("\t")
+      io.print "\t"
+      kmer_hash.number_of_abundances.times{|i| io.print "\tcoverage#{i+1}"}
+      io.puts
+
+      #write data
+      pos = 1
+      Bio::Sequence::NA.new(trail_sequence).window_search(kmer_hash.kmer_length,1) do |kmer|
+        io.puts [trail_name, pos, kmer_hash[kmer]].flatten.join("\t")
+        pos += 1
+      end
+    end
+  end
+end

data/lib/assembly/node_finder.rb

@@ -0,0 +1,171 @@
+module Bio
+  module AssemblyGraphAlgorithms
+    class NodeFinder
+      include Bio::FinishM::Logging
+
+      # Find the node whre the read with the given sequence_id (sequence ID from within velvet)
+      # resides. Assumes that read tracking was turned on during velvetg command.
+      # Returns [node_id, direction] where direction is true if the read is forward-facing
+      # relative to the node, or nil if no node could be found.
+      # When multiple nodes contain the read, return the node that is closest to the beginning
+      # of the read.
+#      def find_unique_node_with_sequence_id(graph, sequence_id)
+#        nodes_with_read = graph.nodes.select do |node|
+#          node.short_reads.select{|r| r.read_id == sequence_id}.length > 0
+#        end
+#        log.debug "Found #{nodes_with_read.length} nodes with the anchor read in it: #{nodes_with_read.collect{|n| n.node_id}.sort.join(',')}"
+#        return nil if nodes_with_read.empty?
+#
+#        # TODO: There is a slight bit of imperfection here - multiple nodes can be the minimum
+#        # Hopefully won't be a bit problem
+#        best_node = nodes_with_read.min do |n1, n2|
+#          r1 = n1.short_reads.find{|r| r.read_id == sequence_id}
+#          r2 = n2.short_reads.find{|r| r.read_id == sequence_id}
+#          r1.start_coord <=> r2.start_coord
+#        end
+#        best_noded_read = best_node.short_reads.find{|r| r.read_id == sequence_id}
+#        return best_node, best_noded_read.direction
+#      end
+
+      #
+      def find_unique_nodes_with_sequence_ids(graph, sequence_ids, options={})
+        # Create data structure
+        endings = {}
+        sequence_ids.each do |seq_id|
+          endings[seq_id] = []
+        end
+
+        # Fill data structure with candidate nodes
+
+        graph.nodes.each do |node|
+          next if options[:target_node_ids] and !options[:target_node_ids].include?(node.node_id)
+          next if node.short_reads.nil?
+          node.short_reads.each do |read|
+            if endings[read.read_id]
+              endings[read.read_id].push node
+            end
+          end
+        end
+
+        # Pick the best node from each of the candidate nodes for each sequence_id
+        return endings.collect do |sequence_id, nodes|
+          pick_best_node_for_read_id(sequence_id, nodes)
+        end
+      end
+
+      # Pick the best node from each of the candidate nodes a sequence_id, and
+      # return an array [best_node, best_noded_read.direction, best_noded_read] or
+      # [] if no probe could be found
+      def pick_best_node_for_read_id(sequence_id, nodes)
+        best_node = nodes.min do |n1, n2|
+          r1 = n1.short_reads.find{|r| r.read_id == sequence_id}
+          r2 = n2.short_reads.find{|r| r.read_id == sequence_id}
+          r1.start_coord <=> r2.start_coord
+        end
+        if best_node
+          best_noded_read = best_node.short_reads.find{|r| r.read_id == sequence_id}
+          [best_node, best_noded_read.direction, best_noded_read]
+        else
+          []
+        end
+      end
+
+      # Pick the best nodes from each of the probe sequences, and
+      # return an array for each probe sequence:
+      # ( [best_node, best_noded_read.direction, best_noded_read] or
+      # [] if no probe could be found )
+      def find_probes_from_read_to_node(graph, read_to_node, probe_sequence_ids)
+        return probe_sequence_ids.collect do |sequence_id|
+          nodes = read_to_node[sequence_id].collect{|node_id| graph.nodes[node_id]}
+          pick_best_node_for_read_id(sequence_id, nodes)
+        end
+      end
+
+      def find_unique_node_with_kmers(velvet_graph, kmers)
+        # TODO: search in a more sane way, algorithmically
+        # TODO: only choose kmers that are unique to the assembly
+        found_node = nil
+        found_direction = nil
+        kmers.each_with_index do |fwd, i|
+          rev = Bio::Sequence::NA.new(fwd).reverse_complement.to_s.upcase
+          #log.debug "Testing kmer #{i} #{fwd} / #{rev}"
+          current_haul = []
+          velvet_graph.nodes.each do |node|
+            if log.debug?
+              str = "Searching for #{fwd} and #{rev} in node #{node.node_id}"
+            end
+            if node.sequence? and (node.sequence.include?(fwd) or node.sequence.include?(rev))
+              current_haul.push node
+            end
+          end
+          if !current_haul.empty?
+            if current_haul.length == 1
+              node = current_haul[0]
+              log.debug "Found a possibly suitable node in the graph, of length #{node.length}" if log.debug?
+              found_node = node
+              if found_node.sequence.include?(fwd)
+                if found_node.sequence.include?(rev)
+                  log.debug "Found a kmer that is included in the forward and reverse directions of the same node. Unlucky, ignoring this kmer" if log.debug?
+                else
+                  log.info "Found a suitable kmer fwd: #{fwd}"
+                  found_direction = true
+                end
+              else
+                log.info "Found a suitable kmer rev: #{rev}"
+                found_direction = false
+              end
+              break unless found_direction.nil?
+            else
+              log.debug "kmer #{fwd}/#{rev} was found in multiple nodes, so not using it as a starting/ending node" if log.debug?
+            end
+          end
+        end
+        return found_node, found_direction
+      end
+
+      def find_nodes_with_kmers(velvet_graph, kmers)
+        # TODO: search in a more sane way, algorithmically
+        # TODO: only choose kmers that are unique to the assembly
+        found_nodes = []
+        kmers.each_with_index do |fwd, i|
+          rev = Bio::Sequence::NA.new(fwd).reverse_complement.to_s.upcase
+          #log.debug "Testing kmer #{i} #{fwd} / #{rev}"
+          current_haul = []
+          velvet_graph.nodes.each do |node|
+            if log.debug?
+              str = "Searching for #{fwd} and #{rev} in node #{node.node_id}"
+              #str += node.sequence
+            end
+            if node.sequence? and (node.sequence.include?(fwd) or node.sequence.include?(rev))
+              current_haul.push node
+            end
+          end
+          if !current_haul.empty?
+            if current_haul.length == 1
+              node = current_haul[0]
+              log.debug "Found a possibly suitable node in the graph, of length #{node.length}" if log.debug?
+              found_node = node
+              if found_node.sequence.include?(fwd)
+                if found_node.sequence.include?(rev)
+                  log.debug "Found a kmer that is included in the forward and reverse directions of the same node. Unlucky, ignoring this kmer" if log.debug?
+                  found_nodes = [found_node]
+                else
+                  log.debug "Found a suitable kmer fwd: #{fwd}"
+                  found_nodes = [found_node]
+                end
+              else
+                log.debug "Found a suitable kmer rev: #{rev}"
+                found_nodes = [found_node]
+              end
+              break
+            else
+              log.debug "kmer #{fwd}/#{rev} was found in multiple nodes" if log.debug?
+              found_nodes = current_haul
+            end
+          end
+        end
+        return found_nodes
+      end
+    end
+  end
+end