finishm 0.0.1

data/lib/assembly/coverage_based_graph_filter.rb

@@ -0,0 +1,68 @@
+require 'set'
+
+module Bio::AssemblyGraphAlgorithms
+  class CoverageBasedGraphFilter
+    include Bio::FinishM::Logging
+
+    # Remove all nodes from the graph that do not have sufficient coverage
+    # (i.e. possibly are sequencing error artefacts)
+    #
+    # Options:
+    # :whitelisted_sequences: provide an enumerable of sequence IDs, don't remove any nodes that have reads tracked to any of these IDs
+    #
+    # Returns nodes_removed, arcs_removed (as objects, in particular order)
+    def remove_low_coverage_nodes(graph, threshold, options = {})
+      graph.delete_nodes_if do |node|
+        deleting = false
+        if node.coverage and (node.coverage < threshold)
+          deleting = true
+        end
+
+        if deleting and options[:whitelisted_sequences] and !node.short_reads.nil?
+          options[:whitelisted_sequences].each do |seq_id|
+            if node.short_reads.collect{|r| r.read_id}.include?(seq_id)
+              deleting = false
+              log.debug "Preserving low coverage but whitelisted node: #{node.node_id}" if log.debug?
+            end
+          end
+        end
+        deleting
+      end
+    end
+  end
+
+  class ConnectivityBasedGraphFilter
+    include Bio::FinishM::Logging
+
+    # Remove parts of the graph that are unconnected to any whitelisted nodes
+    #
+    # options:
+    # :leash_length: don't explore more than this length away from each of the whitelisted_nodes. Defualt nil, no bounds
+    def remove_unconnected_nodes(graph, whitelisted_nodes, options={})
+      # Copy the whitelist
+      all_whitelisted_nodes = Set.new whitelisted_nodes
+
+      dij = Bio::AssemblyGraphAlgorithms::Dijkstra.new
+      dij_options = {:ignore_directions => true}
+      dij_options[:leash_length] = options[:leash_length]
+
+      # Depth-first search of all the connected parts looking for nodes to keep
+      whitelisted_nodes.each do |originally_whitelisted_node|
+        onode = Bio::Velvet::Graph::OrientedNodeTrail::OrientedNode.new
+        onode.node = originally_whitelisted_node
+        onode.first_side = Bio::Velvet::Graph::OrientedNodeTrail::START_IS_FIRST #irrelevant which is first because :ignore_directions => true
+        log.debug "Testing for connectivity from #{onode.node.node_id}" if log.debug?
+
+        min_distances = dij.min_distances(graph, onode, dij_options)
+        min_distances.each do |key, distance|
+          all_whitelisted_nodes << graph.nodes[key[0]]
+        end
+      end
+
+      # Delete all nodes that aren't in the
+      graph.delete_nodes_if do |node|
+        !all_whitelisted_nodes.include?(node)
+      end
+    end
+  end
+end

data/lib/assembly/depth_first_search.rb

@@ -0,0 +1,63 @@
+require 'ds'
+require 'set'
+
+module Bio
+  module Velvet
+    class Graph
+      # Do a depth first search starting at this oriented node,
+      # yielding an OrientedNodeTrail at each new node encountered.
+      # The new node is the last node in the yielded trail. The result
+      # of the yield tells whether this method whether to abandon
+      # searching further from this point (false)
+      # or keep going (true).
+      def depth_first_search(oriented_node)
+        log = Bio::Log::LoggerPlus['finishm']
+        discovered_oriented_nodes = Set.new
+
+        # Traverse through the graph, yielding at each new node
+        current_path = Bio::Velvet::Graph::OrientedNodeTrail.new
+        current_path.add_oriented_node oriented_node
+
+        stack = DS::Stack.new
+        stack.push current_path
+
+        # While there is more on the stack
+        while current_path = stack.pop
+          log.debug "Perhaps #{current_path.last}?" if log.debug?
+          if discovered_oriented_nodes.include?(path_to_searchable(current_path))
+            # Already seen this node, do nothing with it
+            log.debug "Skipping #{current_path.last} since that has already been seen" if log.debug?
+            next
+          else
+            log.debug "That's a new node, #{current_path.last}" if log.debug?
+            # Found a new node for the user to play with
+            discovered_oriented_nodes << path_to_searchable(current_path)
+
+            continue = yield current_path
+
+            # prep for next time if required.
+            if continue
+              # Sort node IDs to simplify testing
+              next_nodes = current_path.neighbours_of_last_node(self).sort{|n1, n2|
+                -(n1.node.node_id <=> n2.node.node_id)
+              }
+              next_nodes.each do |n|
+                path = current_path.copy
+                path.add_oriented_node n
+                stack.push path
+              end
+            end
+          end
+        end
+      end
+
+      private
+      # Set#include? doesn't pick up when the same OrientedNode is picked
+      # up twice independently, I don't think. So convert to an array first
+      def path_to_searchable(path)
+        last = path.last
+        return [last.node.node_id, last.first_side]
+      end
+    end
+  end
+end

data/lib/assembly/dijkstra.rb

@@ -0,0 +1,216 @@
+require 'ds'
+require 'set'
+
+class Bio::AssemblyGraphAlgorithms::Dijkstra
+  include Bio::FinishM::Logging
+
+  # Return an array of DistancedOrientedNode objects, those reachable from
+  # the initial_oriented_node. options:
+  # :leash_length => max distance explored,
+  #    can be set to nil to search indefinitely
+  # :ignore_directions: => true or false (default). If true, explore direction-independently.
+  #   i.e. if 1s->3s and 2s->3s, then include 2s in the returned set of min_distances
+  #   and continue exploring from 2s. Return each found node twice, once for each direction
+  # :neighbour_finder => an object that responds to #neighbours(oriented_node) and
+  #   returns an array of Bio::FinishM::PairedEndNeighbourFinder::Neighbour objects
+  #   default: just search using OrientedNode#next_neighbours
+  # :max_nodes => maximum number of nodes to return, to prevent out of control
+  #   exploring of the graph. If there is plenty of nodes to explore, then the
+  #   length of the returned hash is options[:max_nodes]+1 (+1 because the starting
+  #   node is included). It will probably be longer if :ignore_directions == true, in that
+  #   case the number of node_ids is constrained. It may also be longer if there is ties
+  #   at the edges of the constrained exploration.
+  #
+  # Returns a Hash of [node_id, first_side] => distance
+  def min_distances(graph, initial_oriented_node, options={})
+    pqueue = DS::AnyPriorityQueue.new {|a,b| a < b}
+    first = DistancedOrientedNode.new
+    first.node = initial_oriented_node.node
+    first.first_side = initial_oriented_node.first_side
+    first.distance = 0
+    pqueue.push first, first.distance
+
+    to_return = {}
+    first_node = true
+    found_nodes = Set.new([first.node.node_id])
+
+    while min_distanced_node = pqueue.shift
+
+      # Add/overwrite the current one
+      to_return[min_distanced_node.to_settable] = min_distanced_node.distance
+
+      log.debug "Working from #{min_distanced_node.inspect}" if log.debug?
+
+      if options[:leash_length] and min_distanced_node.distance > options[:leash_length]
+        # we are passed leash length, and this is the nearest node. So we are finito.
+        log.debug "passed the leash length, cutting short our travels" if log.debug?
+        break
+      end
+
+      if options[:max_nodes] and found_nodes.length > options[:max_nodes]
+        log.debug "passed max-nodes threshold and have #{found_nodes.length} nodes" if log.debug?
+        # remove extras that may have been queued if we are over the limit
+        distances_direction_agnostic = {}
+        to_return.each do |key, distance|
+          prev = distances_direction_agnostic[key[0]]
+          if prev.nil? or prev > distance
+            distances_direction_agnostic[key[0]] = distance
+          end
+        end
+        if distances_direction_agnostic.length > options[:max_nodes]
+          sorted = distances_direction_agnostic.to_a.sort{|a,b| a[1]<=>b[1]}
+          # deal with ties i.e. at the edge there can be multiple neighbours
+          last_distance = sorted[options[:max_nodes]][1]
+
+          # only keep those nodes that are sufficiently close
+          to_return.select! do |key, distance|
+            distance <= last_distance
+          end
+        end
+        break
+      end
+
+      # Queue nodes after this one
+      current_distance = min_distanced_node.distance
+
+      # Find neighbouring nodes
+      neighbours = nil
+      if options[:neighbour_finder]
+        neighbours = options[:neighbour_finder].neighbours(min_distanced_node)
+      else
+        neighbours = min_distanced_node.next_neighbours(graph)
+      end
+
+      # explore each neighbour node
+      neighbours.each do |onode|
+        found_nodes << onode.node.node_id
+        new_distance = current_distance
+        if options[:neighbour_finder]
+          # Don't use negative distances as this algorithm cannot handle it, and it is impossible
+          # anyway
+          if onode.distance > 0
+            new_distance += onode.distance
+          else
+            new_distance += 0
+          end
+        end
+        unless first_node
+          new_distance += min_distanced_node.node.length_alone
+        end
+
+        if to_return[onode.to_settable] and to_return[onode.to_settable] <= new_distance
+          # We already know a shorter path to this neighbour, so ignore it
+          log.debug "Already seen this node at the same or shorter distance, going no further" if log.debug?
+        else
+          log.debug "Queuing new distance for neighbour: #{onode}: #{new_distance}" if log.debug?
+          # new shortest distance found. queue it up
+          distanced_node = DistancedOrientedNode.new
+          distanced_node.node = onode.node
+          distanced_node.first_side = onode.first_side
+          distanced_node.distance = new_distance
+          to_return[onode.to_settable] = new_distance
+          pqueue.push distanced_node, new_distance
+
+          if options[:ignore_directions]
+            reverse = DistancedOrientedNode.new
+            reverse.node = onode.node
+            reverse.first_side = onode.reverse.first_side
+            reverse.distance = new_distance
+            to_return[onode.to_settable] = new_distance
+            pqueue.push reverse, new_distance
+          end
+        end
+      end
+
+      first_node = false
+    end
+
+    # if ignore directions, then fixup the return so that each direction is included
+    if options[:ignore_directions]
+      new_to_return = {}
+      to_return.each do |key, distance|
+        keys = [
+          Bio::Velvet::Graph::OrientedNodeTrail::START_IS_FIRST,
+          Bio::Velvet::Graph::OrientedNodeTrail::END_IS_FIRST].collect do |direction|
+            [key[0], direction]
+          end
+        new_distance = keys.collect{|k| to_return[k]}.reject{|d| d.nil?}.min
+        keys.each do |key|
+          new_to_return[key] = new_distance
+        end
+      end
+      to_return = new_to_return
+    end
+
+    return to_return
+  end
+
+  # like #min_distances except explores in both directions
+  def min_distances_in_both_directions(graph, node, options={})
+    all_min_distances = {}
+    [
+      Bio::Velvet::Graph::OrientedNodeTrail::START_IS_FIRST,
+      Bio::Velvet::Graph::OrientedNodeTrail::END_IS_FIRST,
+      ].each do |direction|
+        onode = Bio::Velvet::Graph::OrientedNodeTrail::OrientedNode.new(node, direction)
+        min_distances = min_distances(graph, onode, options)
+        min_distances.each do |node_direction, distance|
+          current = all_min_distances[node_direction]
+          unless current and current > distance
+            all_min_distances[node_direction] = distance
+          end
+        end
+      end
+    return all_min_distances
+  end
+
+  def min_distances_from_many_nodes_in_both_directions(graph, nodes, options={})
+    all_min_distances = {}
+    nodes.each do |node|
+    [
+      Bio::Velvet::Graph::OrientedNodeTrail::START_IS_FIRST,
+      Bio::Velvet::Graph::OrientedNodeTrail::END_IS_FIRST,
+      ].each do |direction|
+        onode = Bio::Velvet::Graph::OrientedNodeTrail::OrientedNode.new(node, direction)
+        min_distances = min_distances(graph, onode, options)
+        min_distances.each do |node_direction, distance|
+          current = all_min_distances[node_direction]
+          unless current and current > distance
+            all_min_distances[node_direction] = distance
+          end
+        end
+      end
+    end
+    return all_min_distances
+  end
+
+  # An oriented node some distance from the origin of exploration
+  class DistancedOrientedNode
+    attr_accessor :node, :first_side, :distance
+
+    # Using Set object, often we want two separate objects to be considered equal even if
+    # they are distinct objects
+    def to_settable
+      [@node.node_id, @first_side]
+    end
+
+    # Which side of the node is not first?
+    def second_side
+      @first_side == Bio::Velvet::Graph::OrientedNodeTrail::START_IS_FIRST ?
+      Bio::Velvet::Graph::OrientedNodeTrail::END_IS_FIRST :
+        Bio::Velvet::Graph::OrientedNodeTrail::START_IS_FIRST
+    end
+
+    def next_neighbours(graph)
+      onode = Bio::Velvet::Graph::OrientedNodeTrail::OrientedNode.new
+      onode.node = @node
+      onode.first_side = @first_side
+      return onode.next_neighbours(graph)
+    end
+
+    def inspect
+      "DistancedOrientedNode #{object_id}: node=#{@node.node_id} first=#{@first_side} distance=#{@distance}"
+    end
+    alias_method :to_s, :inspect
+  end
+end

data/lib/assembly/fluffer.rb

@@ -0,0 +1,253 @@
+require 'ds'
+require 'set'
+
+module Bio
+  module AssemblyGraphAlgorithms
+    class Fluffer
+      include Bio::FinishM::Logging
+      BEYOND_LEASH_LENGTH_FATE = :beyond_leash_length
+      TERMINAL_NODE_FATE = :terminal_node
+      DEAD_END_FATE = :dead_end
+
+
+      class FlufferHalfResult
+        attr_accessor :golden_paths, :golden_path_fragments, :golden_path_fates
+
+        def initialize
+          @golden_paths = []
+          @golden_path_fragments = []
+        end
+      end
+
+      # Holds an array of paths, but also the fates of each of those paths - why were they halted?
+      class FlufferPathSet < Array
+        attr_accessor :fates
+
+        def initialize
+          @fates = []
+        end
+      end
+
+      # Return an array of array of paths.
+      def fluff(finishm_graph, leash_length, options={})
+        log.debug "Fluffing part 1.." if log.debug?
+        half_results = fluff_part1(finishm_graph, leash_length, options)
+
+        log.debug "Found fluff half results: #{half_results}" if log.debug?
+        log.debug "Fluffing part 2.." if log.debug?
+        return fluff_part2(half_results)
+      end
+
+      def fluff_part1(finishm_graph, leash_length, options={})
+        # Get a set of all probes so that they can be checked against
+        log.debug "Found #{finishm_graph.probe_nodes.reject{|node| node.nil?}.length} different probes that were in the final velvet graph" if log.debug?
+        half_results = []
+        graph = finishm_graph.graph
+
+        # For each probe in the graph
+        finishm_graph.probe_nodes.each_with_index do |node, probe_index|
+
+          # If the node is not found in the graph, then forget it
+          if node.nil?
+            half_results.push FlufferHalfResult.new
+
+          else # probe was found in the graph. Start finding them paths.
+            # start exploring the squid way
+            golden_paths = [] #These is the full paths found
+            golden_fragments = [] #paths to join up to the end
+            already_visited_nodes = Set.new #Nodes that have already been explored
+            golden_path_fates = [] #An array of how each of the paths were halted
+
+            golden_onodes = Set.new #Nodes that stop exploration
+            terminal_nodes = Set.new
+            # Add all the nodes that are being probed, because we don't want double exploration
+            # i.e. want probe1 => probe2, probe2 => probe3, but not probe1 => probe2 => probe3
+            finishm_graph.probe_nodes.each_with_index do |node, probe_index2|
+              # Don't add the node itself. This is a special case which is already handled below
+              unless probe_index == probe_index2 or node.nil?
+                terminal_nodes << finishm_graph.velvet_oriented_node(probe_index2).to_settable
+              end
+            end
+
+            stack = DS::Stack.new
+            stack.push finishm_graph.initial_path_from_probe(probe_index)
+
+            while current_path = stack.pop
+              log.debug "Perhaps #{current_path}?" if log.debug?
+              if golden_onodes.include?(current_path.last.to_settable)
+                # Probably a golden fragment, unless the node found is in the current path.
+                # if that is true, that's a loop along a golden path.
+                first_index = nil
+                current_path.each_with_index do |directed_node, i|
+                  if directed_node.node == current_path.last.node and
+                    directed_node.first_side == current_path.last.first_side
+                    first_index = i
+                    break
+                  end
+                end
+
+                if first_index == current_path.length-1
+                  # Found another golden path(s)
+                  log.debug "Ran into a golden node" if log.debug?
+                  golden_fragments.push current_path
+                  current_path.each do |onode|
+                    golden_onodes << onode.to_settable
+                  end
+                else
+                  # Loop found along a golden path or fragment
+                  log.debug "Found a loop along a golden path: #{current_path}" if log.debug?
+                  next
+                end
+              elsif already_visited_nodes.include?(current_path.last.to_settable) and
+                !terminal_nodes.include?(current_path.last.to_settable)
+                # Already seen this (non-golden) node, do nothing with it
+                log.debug "Skipping #{current_path.last} since that has already been seen" if log.debug?
+                next
+              else
+                if log.debug?
+                  second_last_node = current_path[current_path.length-2]
+                  second_last_node ||= 'initial_node'
+                  log.debug "That's a new node, #{second_last_node}/#{current_path.last}" if log.debug?
+                end
+
+                # if we aren't beyond the leash. Presumably this
+                # doesn't happen much, but there is a possibility that the leash will
+                # prevent a real path being discovered. If there is two or more paths to a node
+                # and a path longer than the leash is discovered first, then all the
+                # nodes on that leash will be marked as discovered when they really aren't
+                # TODO: fix the above bug
+                if leash_length.nil? or current_path.length_in_bp < leash_length
+                  # Found a new node for the user to play with
+                  already_visited_nodes << current_path.last.to_settable
+
+                  # Have we found a path to one of the other probes?
+                  if terminal_nodes.include?(current_path.last.to_settable)
+                    log.debug "Found the terminal node: #{current_path}" if log.debug?
+                    golden_paths.push current_path
+                    golden_path_fates.push TERMINAL_NODE_FATE
+
+                  else # Use an else statement here because we want exploration to stop when other probes are encountered
+
+                    # prep for next time
+                    # Sort node IDs to simplify testing
+                    next_nodes = current_path.neighbours_of_last_node(graph).sort{|n1, n2|
+                      -(n1.node.node_id <=> n2.node.node_id)
+                    }
+                    next_nodes.each do |n|
+                      path = current_path.copy
+                      path.add_oriented_node n
+                      log.debug "Pushing a path yet to be explored: #{path}" if log.debug?
+                      stack.push path
+                    end
+
+                    # If we are at a dead end, add this whole path as a golden path. This
+                    # is low coverage fluff, perhaps. Or it is nothing.
+                    if next_nodes.empty?
+                      log.debug "Found a dead end path: #{current_path}" if log.debug?
+                      golden_paths.push current_path
+                      golden_path_fates.push DEAD_END_FATE
+                      current_path.each do |onode|
+                        golden_onodes << onode.to_settable
+                      end
+                    end
+                  end
+                else
+                  if log.debug?
+                    log.debug "Found a path that made it to the end of the leash, with path length #{current_path.length_in_bp} vs leash length #{leash_length}"
+                    log.debug "Path given up on: #{current_path}"
+                    log.debug "Path sequence given up on: #{current_path.sequence}"
+                    log.debug "Node lengths: #{current_path.collect{|n| n.node.length_alone}.join(',')}"
+                  end
+                  # Record this past-leash-length path
+                  golden_paths.push current_path
+                  golden_path_fates.push BEYOND_LEASH_LENGTH_FATE
+                  current_path.each do |onode|
+                    golden_onodes << onode.to_settable
+                  end
+                end
+              end
+            end
+
+            log.debug "Found #{golden_paths.length} golden paths and #{golden_fragments.length} golden fragments" if log.debug?
+            fluff_half_result = FlufferHalfResult.new
+            fluff_half_result.golden_paths = golden_paths
+            fluff_half_result.golden_path_fragments = golden_fragments
+            fluff_half_result.golden_path_fates = golden_path_fates
+
+            half_results.push fluff_half_result
+          end
+        end
+
+        return half_results
+      end
+
+      def fluff_part2(half_results)
+        all_all_paths = []
+
+        half_results.each do |segment_half_result|
+          # OK, so we've transformed the data into a state where there is
+          # at least one path through the data
+          # and tentacles hanging off various golden nodes.
+          # Now separate out the paths and return the array.
+          # First transform the data so it can be referenced by the end node
+          terminal_golden_nodes_to_paths = {}
+          segment_half_result.golden_path_fragments.each do |fragment|
+            l = fragment.last.to_settable
+            terminal_golden_nodes_to_paths[l] ||= []
+            terminal_golden_nodes_to_paths[l].push fragment
+          end
+          # Next backtrack through the paths
+          # Each path starts at the beginning and ends at a
+          # golden node
+          all_paths = FlufferPathSet.new
+          stack = DS::Stack.new
+          # Push the golden path and all paths that finish at the last node
+          segment_half_result.golden_paths.each_with_index do |golden_path, i|
+            stack.push [golden_path, 0, segment_half_result.golden_path_fates[i]]
+          end
+
+          while array = stack.pop
+            current_path = array[0]
+            num_to_ignore = array[1]
+            fate = array[2]
+
+            log.debug "Defragging #{current_path.to_s}/#{fate}, ignoring the last #{num_to_ignore} nodes" if log.debug?
+            all_paths.push current_path
+            all_paths.fates.push fate
+
+            # Iterate down this path, spawning new paths if there
+            # are paths that intersect
+            passed_nodes = []
+            current_path.trail.reverse.each_with_index do |onode, i|
+              unless i < num_to_ignore #ignore the last one(s) because they've already been handled
+                frags = terminal_golden_nodes_to_paths[onode.to_settable]
+                log.debug "Offshoots from #{onode}: #{frags.nil? ? '[]' : frags.collect{|f| f.collect{|n| n.node_id}.join(',')}.join(' and ')}" if log.debug?
+                if frags
+                  frags.each do |fragment|
+                    log.debug "Using an offshoot: #{fragment.to_s}" if log.debug?
+                    # The fragment extends from the beginning to the golden node,
+                    # the current node. So create a new complete path,
+                    # And push it to the stack.
+                    new_golden = fragment.copy
+                    log.debug "Adding #{new_golden.to_s} and #{passed_nodes.collect{|n| n.node_id}}" if log.debug?
+                    passed_nodes.reverse.each_with_index do |onode, i|
+                      new_golden.add_oriented_node onode
+                    end
+                    log.debug "Enqueueing: #{new_golden.to_s} ignoring the last #{i+1} nodes" if log.debug?
+                    stack.push [new_golden, i+1, fate]
+                  end
+                end
+              end
+              passed_nodes.push onode
+            end
+          end
+
+          # All the paths are in an array, just a linear series of distinct paths
+          all_all_paths.push all_paths
+        end
+
+        return all_all_paths
+      end
+    end
+  end
+end