finishm 0.0.1

data/lib/assembly/paired_end_assembler.rb

@@ -0,0 +1,53 @@
+class Bio::AssemblyGraphAlgorithms::PairedEndAssembler < Bio::AssemblyGraphAlgorithms::SingleEndedAssembler
+  include Bio::FinishM::Logging
+
+  # Assemble considering reads as a possibly paired-ended. Options are as per SingleEndedAssembler#assemble_from,
+  # with the addition of
+  # :min_insert_size: minimum length of fragment pair required to satisfy the additional
+  # constraints of the paired end assembler.
+  # :max_insert_size: maximum length of fragment pair.
+  def assemble_from(initial_path, visited_nodes)
+    visited_nodes = Set.new
+    while true
+      # Try to assemble using single ended techniques first, and only if that fails fall
+      # back to paired-end techniques.
+      path, visited_nodes, next_neighbours = super(initial_path, visited_nodes)
+
+      # The next_neighbours
+      if next_neighbours.empty?
+        # No-where to go, do nothing
+        # TODO: try to jump over the gap using paired-end sequences
+      elsif next_neighbours.length < 2
+        raise "Programming error"
+      else
+        # Choose between forks based on paired-end data
+        next_neighbours.select! do |oneigh|
+          confirm_connection_backwards(oneigh, path)
+        end
+      end
+    end
+  end
+
+  # Return true if the backward connection is strong enough to warrant adding
+  # the new_node to the current_path. Else return false
+  #
+  # In order to qualify as a warranted path, reads from the pair must have
+  # at least 1 connection backwards at least options[:min_insert_size] backwards
+  # from the end of the current path, but not more than options[:max_insert_size]
+  # Connection length is the length of the read pair's insert size.
+  def confirm_connection_backwards(new_onode, current_path, min_insert_size, max_insert_size)
+    min_insert_size = @assembly_options[:min_insert_size]
+    max_insert_size = @assembly_options[:max_insert_size]
+
+    # Collect the reads that would qualify if they were in the new node
+    qualifying_reads = Set.new
+    current_path.reverse.each do |node|
+
+    end
+
+    # Look at all the sequences in the new onode. Do any of them qualify
+    new_onode.node.short_reads.each do |short|
+
+    end
+  end
+end

data/lib/assembly/paired_end_neighbour_finder.rb

@@ -0,0 +1,176 @@
+class Bio::FinishM::PairedEndNeighbourFinder
+  include Bio::FinishM::Logging
+
+  # parameters for exploration (and code optimisation, actually)
+  attr_accessor :min_adjoining_reads, :max_adjoining_node_coverage
+
+  def initialize(finishm_graph, insert_size)
+    @finishm_graph = finishm_graph
+    @insert_size = insert_size
+  end
+
+  # Return an array of Neighbour objects that are adjoined either directly through the
+  # de-Bruijn graph or through paired-end connections
+  def neighbours(oriented_node)
+    direct_neighbours = oriented_node.next_neighbours(@finishm_graph.graph)
+    paired_neighbours = paired_neighbour_distances(oriented_node)
+
+    # Return a dereplicated set, prefer direct connections to paired connections
+    dereplicated = {}
+    direct_neighbours.each do |oneigh|
+      key = oneigh.node_id
+      raise if dereplicated[key]
+      n = Neighbour.new
+      n.node = oneigh.node
+      n.first_side = oneigh.first_side
+      n.connection_type = Neighbour::DIRECT_CONNECTION
+      n.distance = 0
+      dereplicated[key] = n
+    end
+    paired_neighbours.each do |n|
+      key = n.node.node_id
+      next if dereplicated[key] #skip those already connected by direct connection
+      dereplicated[key] = n
+    end
+    return dereplicated.values
+  end
+
+  # Return an array of Neighbour objects representing nodes that has reads which
+  # are paired with reads of the given node. Each distance is the estimated
+  # distance separating the nodes.
+  def paired_neighbour_distances(oriented_node)
+    # Collect a list of node IDs that have a sufficient number of connections
+    #hash of node ID to Array of [first_read, second_read_id] where first_read is on the current
+    # node and second_read is on the neighbour
+    neighbour_node_read_pairs = {}
+    oriented_node.node.short_reads.each do |read|
+      # skip reads not going in the direction consistent with direction of travel
+      next unless (read.direction == true and oriented_node.first_side == Bio::Velvet::Graph::OrientedNodeTrail::START_IS_FIRST) or
+      (read.direction == false and oriented_node.first_side == Bio::Velvet::Graph::OrientedNodeTrail::END_IS_FIRST)
+
+      pair_read_id = @finishm_graph.velvet_sequences.pair_id(read.read_id)
+      unless pair_read_id.nil? #i.e. if read is paired
+        @finishm_graph.read_to_nodes[pair_read_id].each do |node_id|
+          neighbour_node_read_pairs[node_id] ||= []
+          neighbour_node_read_pairs[node_id] << [read, pair_read_id]
+        end
+      end
+    end
+
+    # Create a hash of paired node_ids to [fwd_noded_read, rev_noded_read]
+    node_ids_to_read_and_pair = {}
+    neighbour_node_read_pairs.each do |neighbour_node_id, pairs|
+      next if (@min_adjoining_reads and pairs.length < @min_adjoining_reads)
+
+      # ignore neighbours that have too much coverage, these are likely
+      # to be incorrect connections e.g. adapter contamination
+      neighbour_node = @finishm_graph.graph.nodes[neighbour_node_id]
+      log.debug "Found neighbour node coverage #{neighbour_node.coverage}" if log.debug?
+      if @max_adjoining_node_coverage and neighbour_node.coverage > @max_adjoining_node_coverage
+        log.debug "Skipping node #{neighbour_node} as it has coverage #{neighbour_node.coverage} not < #{@max_adjoining_node_coverage}" if log.debug?
+        next
+      end
+
+      node_ids_to_read_and_pair[neighbour_node_id] ||= []
+
+      pairs.each do |pair|
+        read = pair[0]
+        pair_read_id = pair[1]
+        rev_read = neighbour_node.short_reads.get_read_by_id(pair_read_id)
+        if rev_read.nil?
+          raise "unexpectedly didn't find read attached to node when one was expected: #{pair_read_id}"
+        end
+        node_ids_to_read_and_pair[neighbour_node_id] << [read, rev_read]
+      end
+    end
+
+    # Collate each list of reads into a list of PairedNeighbour objects
+    neighbours = []
+    node_ids_to_read_and_pair.each do |neighbour_node_id, read_pairs|
+      next if neighbour_node_id == oriented_node.node.node_id #nodes paired to themselves don't count
+
+      neighbour = Neighbour.new
+      neighbour.connection_type = Neighbour::PAIRED_END_CONNECTION
+      neighbour.node = @finishm_graph.graph.nodes[neighbour_node_id]
+      log.debug "Setting neighbour node as #{neighbour.node}" if log.debug?
+
+      # find the expected direction of the
+      direction_vote = {}
+      read_pairs.each do |pair|
+        key = pair[1].direction
+        direction_vote[key] ||= 0
+        direction_vote[key] += 1
+      end
+      found_direction = direction_vote.max{|a,b| a[1] <=> b[1]}[0]
+      if found_direction == true
+        neighbour.first_side = Bio::Velvet::Graph::OrientedNodeTrail::END_IS_FIRST
+      elsif found_direction == false
+        neighbour.first_side = Bio::Velvet::Graph::OrientedNodeTrail::START_IS_FIRST
+      end
+
+      distance_sum = 0
+      num_adjoining_reads = 0
+      read_pairs.each do |pair|
+        if found_direction == pair[1].direction
+          distance = estimate_distance_between_nodes(oriented_node.node, neighbour.node, pair[0], pair[1], @insert_size)
+          unless distance.nil?
+            log.debug "Accepting distance #{distance} from read_id #{pair[1].read_id}" if log.debug?
+            distance_sum += distance
+            num_adjoining_reads += 1
+          end
+        end
+      end
+
+      # don't accept when no adjoining reads are found
+      if num_adjoining_reads > 0 and
+        (@min_adjoining_reads.nil? or num_adjoining_reads >= @min_adjoining_reads)
+
+        if distance_sum < 0
+          # don't predict negative distances
+          neighbour.distance = 0
+        else
+          neighbour.distance = distance_sum.to_f / num_adjoining_reads
+        end
+
+        neighbour.num_adjoining_reads = num_adjoining_reads
+        neighbours.push neighbour
+      end
+    end
+
+    return neighbours
+  end
+
+  # estimate the distance between the two nodes, assuming that the pair orientation is
+  # not problematic. Return nil if the estimated insert size is greater than 2 times the
+  # expected insert size, or less than -1 times the insert size.
+  def estimate_distance_between_nodes(node1, node2, fwd_read, rev_read, insert_size)
+    fwd_contribution = node1.length_alone - fwd_read.offset_from_start_of_node + fwd_read.start_coord
+    rev_contribution = node2.length_alone - rev_read.offset_from_start_of_node + rev_read.start_coord
+    diff = insert_size - fwd_contribution - rev_contribution
+    if diff > insert_size*2 or diff < -insert_size
+      return nil
+    else
+      return diff
+    end
+  end
+
+  class Neighbour < Bio::Velvet::Graph::OrientedNodeTrail::OrientedNode
+    PAIRED_END_CONNECTION = :paired_end_connection
+    DIRECT_CONNECTION = :direct_connection
+    attr_accessor :connection_type
+
+    attr_accessor :node
+    attr_accessor :first_side
+    attr_accessor :distance
+    attr_accessor :num_adjoining_reads
+
+    def to_settable
+      [@node.node_id, @first_side]
+    end
+
+    def inspect
+      "Neighbour #{object_id}: node=#{@node.node_id} first=#{@first_side} distance=#{@distance} num_adjoining_reads=#{@num_adjoining_reads} connection_type:#{@connection_type}"
+    end
+    alias_method :to_s, :inspect
+  end
+end

data/lib/assembly/probed_graph.rb

@@ -0,0 +1,105 @@
+class Bio::FinishM::ProbedGraph
+  include Bio::FinishM::Logging
+  attr_accessor :probe_nodes, :probe_node_directions, :probe_node_reads, :graph
+
+  attr_accessor :velvet_result_directory
+
+  # Most likely a BinarySequenceStore
+  attr_accessor :velvet_sequences
+
+  # Most likely a ReadToNode object
+  attr_accessor :read_to_nodes
+
+  # Were all the probe recovered through the process?
+  def completely_probed?
+    !(@probe_nodes.find{|node| node.nil?})
+  end
+
+  def missing_probe_indices
+    missings = []
+    @probe_nodes.each_with_index do |probe, i|
+      missings.push(i+1) if probe.nil?
+    end
+    return missings
+  end
+
+  # Make a Bio::Velvet::Graph::OrientedNodeTrail with just one
+  # step in it - the node that corresponds to the probe_index
+  def initial_path_from_probe(probe_index)
+    initial_path = Bio::Velvet::Graph::OrientedNodeTrail.new
+    node = @probe_nodes[probe_index]
+    raise "No node found for probe #{probe_index}" if node.nil?
+    direction = @probe_node_directions[probe_index]
+
+    way = direction ?
+    Bio::Velvet::Graph::OrientedNodeTrail::START_IS_FIRST :
+      Bio::Velvet::Graph::OrientedNodeTrail::END_IS_FIRST
+    initial_path.add_node node, way
+    return initial_path
+  end
+
+  # Return a Bio::Velvet::Graph::OrientedNodeTrail::OrientedNode
+  # corresponding to the index of the probe and its direction
+  def velvet_oriented_node(probe_index)
+    node = @probe_nodes[probe_index]
+    if node.nil?
+      return nil
+    else
+      return initial_path_from_probe(probe_index)[0]
+    end
+  end
+
+  # The leash is the number of base pairs from the start of the probe,
+  # but the path finding algorithm simply uses the combined length of all
+  # the nodes without reference to the actual probe sequence. So if the
+  # probe is near the end of a long node, then path finding may fail.
+  # So adjust the leash length to account for this (or keep the nil
+  # if the starting_leash_length is nil)
+  def adjusted_leash_length(probe_index, starting_leash_length)
+    return nil if starting_leash_length.nil?
+
+    read = @probe_node_reads[probe_index]
+    return read.offset_from_start_of_node+starting_leash_length
+  end
+
+  # Return a new ProbedGraph that is the same as the current one
+  # except that only probe specified in the given probe_indices enumerable
+  # are accepted
+  def subgraph(probe_indices)
+    to_return = Bio::FinishM::ProbedGraph.new
+    to_return.graph = @graph
+    to_return.velvet_result_directory = @velvet_result_directory
+    to_return.velvet_sequences = @velvet_sequences
+
+    to_return.probe_nodes = []
+    to_return.probe_node_directions = []
+    to_return.probe_node_reads = []
+    probe_indices.each do |i|
+      to_return.probe_nodes.push @probe_nodes[i-1]
+      to_return.probe_node_directions.push @probe_node_directions[i-1]
+      to_return.probe_node_reads.push @probe_node_reads[i-1]
+    end
+
+    return to_return
+  end
+
+  # Return a list of node IDs that are connected through paired-end linkages.
+  # This method probably belongs in the Node class except that that is
+  # in bio-velvet and yet requires sequence_id_to_node_ids_hash. If all reads are
+  # single ended then this method always returns []
+  def paired_nodes(node)
+    to_return_node_ids = Set.new
+    binding.pry
+    log.debug "Found #{node.short_reads.length} short reads associated with node #{node}" if log.debug?
+    node.short_reads.each do |read|
+      pair_read_id = @velvet_sequences.pair_id(read.read_id)
+      unless pair_read_id.nil? #i.e. if read is paired
+        @read_to_nodes[pair_read_id].each do |node_id|
+          to_return_node_ids << node_id
+        end
+      end
+    end
+    # Convert node IDs to node objects and return
+    return to_return_node_ids.to_a
+  end
+end

data/lib/assembly/read_input.rb

@@ -0,0 +1,79 @@
+# A class representing reads or sets of reads to be assembled
+class Bio::FinishM::ReadInput
+  READ_INPUT_SYMBOLS = [
+    :fasta_singles, :fastq_singles, :fasta_singles_gz, :fastq_singles_gz,
+    :interleaved_fasta, :interleaved_fastq, :interleaved_fasta_gz, :interleaved_fastq_gz,
+    :separate_fasta, :separate_fastq, :separate_fasta_gz, :separate_fastq_gz,
+    ]
+  READ_INPUT_SYMBOLS.each do |sym|
+    attr_accessor sym
+  end
+
+  # Given an OptionParser, add options to it, which parse out read-related options
+  def add_options(option_parser, options)
+    {
+      '--fasta' => :fasta_singles,
+      '--fastq' => :fastq_singles,
+      '--fasta-gz' => :fasta_singles_gz,
+      '--fastq-gz' => :fastq_singles_gz,
+      '--interleaved-fasta' => :interleaved_fasta,
+      '--interleaved-fastq' => :interleaved_fastq,
+      '--interleaved-fasta-gz' => :interleaved_fasta_gz,
+      '--interleaved-fastq-gz' => :interleaved_fastq_gz,
+      '--separate-fasta' => :separate_fasta,
+      '--separate-fastq' => :separate_fastq,
+      '--separate-fasta-gz' => :separate_fasta_gz,
+      '--separate-fastq-gz' => :separate_fastq_gz,
+      }.each do |flag, sym|
+        option_parser.on("#{flag} PATH", Array, "One or more paths to reads, comma separated") do |arg|
+          options[sym] = arg
+        end
+      end
+  end
+
+  # Require at least 1 set of reads to be given, of any type
+  def validate_options(options, argv)
+    return nil if options[:previous_assembly] #bit of a hack, but hey
+    READ_INPUT_SYMBOLS.each do |sym|
+      return nil if options[sym]
+    end
+    return "No definition of reads for assembly was found"
+  end
+
+  # Parse options from options hash into instance variables for this object
+  def parse_options(options)
+    READ_INPUT_SYMBOLS.each do |sym|
+      send("#{sym}=",options[sym]) if options[sym]
+    end
+  end
+
+  # Output a string to be used on the command line with velvet
+  def velvet_read_arguments
+    readset_index = 1
+    args = ''
+    # Have to put probe sequences (which are single-ended) first in this (ordered) hash
+    {
+      :fasta_singles => '-fasta -short',
+      :fastq_singles => '-fastq -short',
+      :fasta_singles_gz => '-fasta.gz -short',
+      :fastq_singles_gz => '-fastq.gz -short',
+      :interleaved_fasta => '-fasta -shortPaired',
+      :interleaved_fastq => '-fastq -shortPaired',
+      :interleaved_fasta_gz => '-fasta.gz -shortPaired',
+      :interleaved_fastq_gz => '-fastq.gz -shortPaired',
+      :separate_fasta => '-fasta -shortPaired -separate',
+      :separate_fastq => '-fastq -shortPaired -separate',
+      :separate_fasta_gz => '-fasta.gz -shortPaired -separate',
+      :separate_fastq_gz => '-fastq.gz -shortPaired -separate',
+      }.each do |sym, velvet_flag|
+        paths = send(sym)
+        unless paths.nil? or paths.empty?
+          args += " #{velvet_flag}"
+          paths.each do |path|
+            args += " #{path}"
+          end
+        end
+      end
+    return args
+  end
+end

data/lib/assembly/read_to_node.rb

@@ -0,0 +1,37 @@
+class Bio::FinishM::ReadToNode
+  include Bio::FinishM::Logging
+
+  def initialize(filename)
+    @bindings = Bio::FinishM::VelvetCBinding.new
+    log.debug "Reading ReadToNode file #{filename}.."
+    raise "Unable to find readToNode binary file" unless File.exist?(filename)
+    @read_to_node = @bindings.read_read_id_to_node_id_lookup_table(filename)
+    log.debug "Finished reading ReadToNode file"
+    @cache = {}
+  end
+
+  # Return an array of node IDs that include the given read id
+  def [](read_id)
+    # cache
+    cache = @cache[read_id]
+    return cache unless cache.nil?
+
+    res = @bindings.get_read_id_to_node_id_indexation(@read_to_node, read_id)
+
+    #   # typedef struct {
+    #   #   IDnum num_nodes;
+    #   #   ReadIdNodeId* read_ids_node_ids;
+    #   # } ReadIdToNodeIdIndexation;
+    #   class ReadIdToNodeIdIndexationStruct < FFI::Struct
+    #     layout :num_nodes, :int32,
+    #     :read_ids_node_ids, :pointer
+    #   end
+    to_return = []
+    structs = FFI::Pointer.new(Bio::FinishM::VelvetCBinding::ReadIdNodeIdStruct, res[:read_ids_node_ids].pointer)
+    0.upto(res[:num_nodes]-1) do |i|
+      to_return << Bio::FinishM::VelvetCBinding::ReadIdNodeIdStruct.new(structs[i])[:node_id].abs
+    end
+    @cache[read_id] = to_return
+    return to_return
+  end
+end