finishm 0.0.1

data/lib/finishm/finisher.rb

@@ -0,0 +1,303 @@
+class Bio::FinishM::Finisher
+  include Bio::FinishM::Logging
+
+  def add_options(opts, options)
+    opts.banner = "\nUsage: finishm finish <options>\n\n"
+
+    options.merge!({
+      :min_leftover_length => false,
+      :kmer_coverage_target => 1,
+      :contig_end_length => 300,
+      :graph_search_leash_length => 20000,
+      :reads_to_assemble => nil,
+    })
+
+    # TODO: make a better interface for this. Maybe specify an entire genome, and then "Contig_1 end, Contig_3 start" or something
+    # Look at the last 300bp of the first contig.
+    extract_exactly_one_contig_from_file = lambda do |fasta_file_path|
+      contig = nil
+      Bio::FlatFile.foreach(Bio::FastaFormat, fasta_file_path) do |e|
+        if contig.nil?
+          contig = e.seq
+        else
+          raise "Multiple sequences found in a contig file! I need exactly one"
+        end
+      end
+      raise "I need a contig to be in the start contig file" if contig.nil?
+      Bio::Sequence::NA.new(contig.to_s)
+    end
+
+    opts.on("--pattern PATTERN", "kmer abundance pattern e.g. '0111001110' [required]") do |arg|
+      options[:pattern] = arg
+    end
+    opts.on("--kmer-abundances FILE", "kmer multiple abundance file [required]") do |arg|
+      options[:kmer_multiple_abundance_file] = arg
+    end
+    opts.on("--upper-threshold NUM", "kmer frequency cutoff to saying 'present' [required]") do |arg|
+      options[:upper_threshold] = arg.to_i
+    end
+    opts.on("--lower-threshold NUM", "kmer frequency cutoff to saying 'not present' [required]") do |arg|
+      options[:lower_threshold] = arg.to_i
+    end
+    opts.on("--reads FILES", "comma-separated list of sequence reads files in the same order as the pattern was supplied [required]") do |arg|
+      options[:reads_files] = arg.split(',').collect{|r| File.absolute_path r}
+    end
+    opts.on("--start-contig FASTA", "path to a fasta file with the starting contig in it (only). Assumes we are building off the end of this contig [required]") do |arg|
+      options[:start_contig] = extract_exactly_one_contig_from_file.call arg
+    end
+    opts.on("--end-contig FASTA", "path to a fasta file with the ending contig in it (only). Assumes we are building onto the start of this contig [required]") do |arg|
+      options[:end_contig] = extract_exactly_one_contig_from_file.call arg
+    end
+
+    opts.separator "\nOptional arguments:\n\n"
+    opts.on("--min-leftover-read-length NUMBER", "when searching for reads with kmers, require the kmer to be at the beginning or end of the selected read [default: #{options[:min_leftover_length]}]") do |arg|
+      options[:min_leftover_length] = arg.to_i
+    end
+    opts.on("--kmer-coverage-target NUMBER", "when searching for reads with kmers, require this many copies per kmer [default: #{options[:kmer_coverage_target]}]") do |arg|
+      options[:kmer_coverage_target] = arg.to_i
+    end
+    opts.on("--already-patterned-reads FILE", "Attempt to assemble the reads in the specified file, useful for re-assembly [default: off]") do |arg|
+      options[:already_patterned_reads] = arg
+    end
+    opts.on("--assembly-png PATH", "Output assembly as a PNG file [default: off]") do |arg|
+      options[:output_graph_png] = arg
+    end
+    opts.on("--assembly-svg PATH", "Output assembly as an SVG file [default: off]") do |arg|
+      options[:output_graph_svg] = arg
+    end
+    opts.on("--assembly-dot PATH", "Output assembly as an DOT file [default: off]") do |arg|
+      options[:output_graph_dot] = arg
+    end
+    opts.on("--assembly-coverage-cutoff NUMBER", "Require this much coverage in each node, all other nodes are removed [default: #{options[:assembly_coverage_cutoff]}]") do |arg|
+      options[:assembly_coverage_cutoff] = arg.to_f
+    end
+    opts.on("--contig-end-length LENGTH", "Number of base pairs to start into the ends of the contigs [default: #{options[:contig_end_length]}]") do |arg|
+      options[:contig_end_length] = arg.to_i
+    end
+
+    Bio::FinishM::GraphGenerator.new.add_options opts, options
+  end
+
+  def validate_options(options, argv)
+    #TODO: give a better description of the error that has occurred
+    if argv.length != 0
+      return "Dangling argument(s) found e.g. #{argv[0]}"
+    elsif options[:already_patterned_reads]
+    else
+      [:upper_threshold,
+      :lower_threshold,
+      :pattern,
+      :kmer_multiple_abundance_file,
+      :reads_files].each do |sym|
+        if options[sym].nil?
+          return "No option found to specify #{sym}"
+        end
+      end
+    end
+    return nil #if here, options all were parsed successfully
+  end
+
+  #TODO: this method is too long - split it up by refactoring
+  def run(options, argv)
+    pooled_reads_filename = 'pooled_sampled_reads.fasta' #TODO: remove this constant into a tempfile or something.
+    if options[:already_patterned_reads] #If skipping read extraction
+      pooled_reads_filename = options[:already_patterned_reads]
+
+    else
+      # Parse pattern from cmdline
+      desired_pattern = KmerAbundancePattern.new
+      desired_pattern.parse_from_human(options[:pattern])
+      if options[:reads_files].length != desired_pattern.length
+        raise "Number of entries in the pattern #{desired_pattern.length} and number of reads files #{options[:reads].length} not equivalent!"
+      end
+
+      # Collect the kmers that will be used to find trusted reads i.e.
+      # Go through each line of the kmer abundance file, looking for kmers that suit the pattern
+      input_file = File.open options[:kmer_multiple_abundance_file]
+      csv = CSV.new(input_file, :col_sep => ' ')
+
+      whitelist_kmers = []
+      blacklist_kmers = []
+      csv.each do |row|
+        max_i = row.length - 2 if max_i.nil?
+
+        kmer = row[0]
+        counts = row[1...row.length].collect{|s| s.to_i}
+        this_pattern = []
+        counts.each_with_index do |count, i|
+          if count > options[:upper_threshold]
+            this_pattern[i] = true
+          elsif count < options[:lower_threshold]
+            this_pattern[i] = false
+          else
+            # coverage was in no man's land between thresholds.
+            # Ignore this kmer as noise.
+            this_pattern[i] = '-'
+          end
+        end
+        #log.debug "Found pattern #{this_pattern} from kmer #{kmer}, which has abundances #{counts}" if log.debug?
+
+        if desired_pattern.consistent_with? this_pattern
+          whitelist_kmers.push row[0]
+        else
+          # kmer is not present when it should be
+          blacklist_kmers.push row[0]
+        end
+      end
+      log.info "After parsing the kmer multiple abundance file, found #{whitelist_kmers.length} kmers that matched the pattern, and #{blacklist_kmers.length} that didn't"
+      unless whitelist_kmers.length > 0
+        log.error "No kmers found that satisfy the given pattern, exiting.."
+        exit 1
+      end
+
+
+      #outdir = options[:output_directory]
+      #Dir.mkdir outdir unless Dir.exist?(outdir)
+
+      # grep the pattern out from the raw reads, subsampling so as to not overwhelm the assembler
+      #Tempfile.open('whitelist') do |white|
+      File.open 'whitelist', 'w' do |white| #TODO: remove 'whitelist' file as a constant
+        white.puts whitelist_kmers.join("\n")
+        white.close
+
+        #Tempfile.open('blacklist') do |black|
+        File.open('black','w') do |black|
+          black.puts blacklist_kmers.join("\n")
+          black.close
+
+          threadpool = []
+          sampled_read_files = []
+          log.info "Extracting reads that contain suitable kmers"
+          options[:reads_files].each_with_index do |file, i|
+            next unless desired_pattern[i] #Don't extract reads from reads where those reads should not have been amplified
+
+            sampled = File.basename(file)+'.sampled_reads.fasta'
+            sampled_read_files.push sampled
+
+            grep_path = "#{ ENV['HOME'] }/git/priner/bin/read_selection_by_kmer " #TODO: this won't work on other people's systems.
+            if options[:min_leftover_length]
+              grep_path += "--min-leftover-length #{options[:min_leftover_length]} "
+            end
+            thr = Thread.new do
+              grep_cmd = "#{grep_path} --whitelist #{white.path} --blacklist #{black.path} --reads #{file} --kmer-coverage-target #{options[:kmer_coverage_target]} > #{sampled}"
+              log.debug "Running cmd: #{grep_cmd}"
+              status, stdout, stderr = systemu grep_cmd
+              log.debug stderr
+
+              raise unless status.exitstatus == 0
+              log.debug "Finished extracting reads from #{file}"
+            end
+            threadpool.push thr
+          end
+          threadpool.each do |thread| thread.join; end #wait until everything is finito
+
+          log.info "Finished extracting reads for sampling. Now pooling sampled reads"
+          pool_cmd = "cat #{sampled_read_files.join ' '} >#{pooled_reads_filename}"
+          log.debug "Running cmd: #{pool_cmd}"
+          status, stdout, stderr = systemu pool_cmd
+          raise stderr if stderr != ''
+          raise unless status.exitstatus == 0
+        end
+      end
+    end
+
+    log.info "Extracting dummy reads from the ends of contigs to use as anchors"
+    start_contig = options[:start_contig]
+    end_contig = options[:end_contig]
+    if [start_contig.length, end_contig.length].min < 2*options[:contig_end_length]
+      log.warn "Choice of initial/terminal nodes to perform graph search with may not be optimal due to the small contig size"
+    end
+    if [start_contig.length, end_contig.length].min < options[:contig_end_length]
+      log.error "At least one contig too small to proceed with current code base, need to fix the code to allow such a small contig"
+      exit 1
+    end
+
+    probe_sequences = [
+      start_contig[start_contig.length-options[:contig_end_length]...start_contig.length],
+      Bio::Sequence::NA.new(end_contig[0...options[:contig_end_length]]).reverse_complement.to_s
+    ]
+    read_input = Bio::FinishM::ReadInput.new
+    read_input.fasta_singles = [pooled_reads_filename]
+    finishm_graph = Bio::FinishM::GraphGenerator.new.generate_graph(probe_sequences, read_input, options)
+    graph = finishm_graph.graph
+    start_node = finishm_graph.probe_nodes[0]
+    start_node_forward = finishm_graph.probe_node_directions[0]
+    end_node = finishm_graph.probe_nodes[1]
+    end_node_forward = finishm_graph.probe_node_directions[1]
+
+    log.info "Node(s) found that are suitable as initial and terminal nodes in the graph search, respectively: #{start_node.node_id} and #{end_node.node_id}"
+
+    log.info "Removing nodes unconnected to either the start or the end from the graph.."
+    original_num_nodes = graph.nodes.length
+    original_num_arcs = graph.arcs.length
+    filter = Bio::AssemblyGraphAlgorithms::ConnectivityBasedGraphFilter.new
+    filter.remove_unconnected_nodes(graph, [start_node, end_node])
+    log.info "Removed #{original_num_nodes-graph.nodes.length} nodes and #{original_num_arcs-graph.arcs.length} arcs"
+
+    if options[:output_graph_png] or options[:output_graph_svg] or options[:output_graph_dot]
+      viser = Bio::Assembly::ABVisualiser.new
+      log.info "Preparing GraphViz object for output"
+      gv = viser.graphviz(graph, {:start_node_id => start_node.node_id, :end_node_id => end_node.node_id})
+
+      if options[:output_graph_png]
+        log.info "Converting assembly to a graphviz PNG #{options[:output_graph_png] }"
+        gv.output :png => options[:output_graph_png], :use => :neato
+      end
+      if options[:output_graph_svg]
+        log.info "Converting assembly to a graphviz SVG #{options[:output_graph_svg] }"
+        gv.output :svg => options[:output_graph_svg], :use => :neato
+      end
+      if options[:output_graph_dot]
+        log.info "Converting assembly to a graphviz DOT #{options[:output_graph_dot] }"
+        gv.output :dot => options[:output_graph_dot]
+      end
+    end
+
+    log.info "Searching for trails between the initial and terminal nodes, within the assembly graph"
+    cartographer = Bio::AssemblyGraphAlgorithms::AcyclicConnectionFinder.new
+    #raise "Untested connection finder below"
+    #trails = cartographer.find_all_trails_between_nodes(graph, start_node, end_node, options[:graph_search_leash_length], start_node_forward)
+    trails = cartographer.find_trails_between_nodes(graph, start_node, end_node, options[:graph_search_leash_length], start_node_forward)
+    log.info "Found #{trails.length} trail(s) between the initial and terminal nodes"
+
+#    log.info "Reading kmer abundances from #{options[:kmer_multiple_abundance_file]}.."
+#    kmer_hash = Bio::KmerMultipleAbundanceHash.parse_from_file options[:kmer_multiple_abundance_file]
+#    log.info "Finished reading the kmer abundances"
+
+#    if options[:trail_kmer_coverage_file]
+#      log.info "Writing out kmer coverages to #{options[:trail_kmer_coverage_file]}.."
+#      writer = Bio::AssemblyGraphAlgorithms::KmerCoverageWriter.new
+#      io = File.open(options[:trail_kmer_coverage_file],'w')
+#      writer.write(io, trails, kmer_hash)
+#      log.info "Finished writing"
+#    end
+
+#    log.info "Filtering trail(s) based on kmer coverage, requiring each kmer in the path to have a minimum of #{options[:kmer_path_filter_min_coverage]} coverage in patterned reads, except for the #{options[:kmer_path_end_exclusion_length]}bp at the ends"
+#    kmer_path_filter = Bio::AssemblyGraphAlgorithms::KmerCoverageBasedPathFilter.new
+#    thresholds = desired_pattern.collect{|c| c == true ? 1 : 0}
+#    log.info "Using thresholds for filtering: #{thresholds}"
+#    trails = kmer_path_filter.filter(trails, kmer_hash, thresholds, :exclude_ending_length => options[:kmer_path_end_exclusion_length])
+#    log.info "After filtering remained #{trails.length} trails"
+
+    printer = Bio::AssemblyGraphAlgorithms::ContigPrinter.new
+    trails.each_with_index do |trail, i|
+      log.debug "Before attachment to the contig, sequence of the trail was #{trail.sequence}" if log.debug?
+      acon = Bio::AssemblyGraphAlgorithms::ContigPrinter::AnchoredConnection.new
+      acon.start_probe_read_id = 1
+      acon.end_probe_read_id = 2
+      acon.start_probe_node = start_node
+      acon.end_probe_node = end_node
+      acon.start_probe_contig_offset = options[:contig_end_length]
+      acon.end_probe_contig_offset = options[:contig_end_length]
+      acon.paths = [trail]
+      log.debug "AnchoredConnection object to print for this trail: #{acon.inspect}" if log.debug?
+
+      puts ">trail#{i+1}"
+      puts printer.one_connection_between_two_contigs(
+        finishm_graph.graph,
+        probe_sequences[0],
+        acon,
+        probe_sequences[1])
+    end
+  end
+end

data/lib/finishm/fluff.rb

@@ -0,0 +1,122 @@
+class Bio::FinishM::Fluff
+  include Bio::FinishM::Logging
+
+  def add_options(optparse_object, options)
+    optparse_object.banner = "\nUsage: finishm fluff --contigs <contig_file> --fastq-gz <reads..> --output-fluff-file <output.fa>
+
+    Takes a set of contigs, and places probes across them (e.g. every 2kb), and then explores the
+    graph from each of these probes, taking all paths within some leash length, including the 'fluff'
+    which is not the same path as along the contig. Prints out all of these paths to a fasta file.\n\n"
+
+    options.merge!({
+      :probe_spacing => 2000,
+      :probe_length => 100,
+      :graph_search_leash_length => 20000,
+    })
+
+    optparse_object.separator "\nRequired arguments:\n\n"
+    optparse_object.on("--contigs FILE", "fasta file containing contigs to find the fluff on [required]") do |arg|
+      options[:contigs_file] = arg
+    end
+    optparse_object.on("--output-fluff-file PATH", "Output found paths to this file in fasta format [required]") do |arg|
+      options[:output_fluff_file] = arg
+    end
+    optparse_object.separator "\nThere must be some definition of reads too:\n\n" #TODO improve this help
+    Bio::FinishM::ReadInput.new.add_options(optparse_object, options)
+
+    optparse_object.separator "\nOptional arguments:\n\n"
+    optparse_object.on("--probe-spacing NUM", Integer, "Distance between probe points in the contig [default: #{options[:probe_spacing]}]") do |arg|
+      options[:probe_spacing] = arg
+    end
+    optparse_object.on("--probe-size NUM", Integer, "Length of the probe to be inserted into the velvet graph. Must be greater than graph kmer length. [default: #{options[:probe_length]}]") do |arg|
+      options[:probe_length] = arg
+    end
+    optparse_object.on("--leash-length NUM", Integer, "Don't explore too far in the graph, only this far and not much more [default: #{options[:graph_search_leash_length]}]") do |arg|
+      options[:graph_search_leash_length] = arg
+    end
+    optparse_object.on("--assembly-png PATH", "Output assembly as a PNG file [default: off]") do |arg|
+      options[:output_graph_png] = arg
+    end
+    optparse_object.on("--assembly-svg PATH", "Output assembly as an SVG file [default: off]") do |arg|
+      options[:output_graph_svg] = arg
+    end
+    optparse_object.on("--assembly-dot PATH", "Output assembly as an DOT file [default: off]") do |arg|
+      options[:output_graph_dot] = arg
+    end
+
+    Bio::FinishM::GraphGenerator.new.add_options optparse_object, options
+  end
+
+  def validate_options(options, argv)
+    #TODO: give a better description of the error that has occurred
+    #TODO: require reads options
+    if argv.length != 0
+      return "Dangling argument(s) found e.g. #{argv[0]}"
+    else
+      [
+        :contigs_file,
+        :output_fluff_file
+      ].each do |sym|
+        if options[sym].nil?
+          return "No option found to specify #{sym}."
+        end
+      end
+
+      unless options[:velvet_kmer_size] < options[:probe_length]
+        return "The probe length must be greater than the kmer length, otherwise it will not be incorporated into the kmer graph"
+      end
+
+      #if return nil from here, options all were parsed successfully
+      return Bio::FinishM::ReadInput.new.validate_options(options, [])
+    end
+  end
+
+  def run(options, argv)
+    # Read in all the contigs sequences
+    probe_sequences = []
+    sequence_names = []
+    Bio::FlatFile.foreach(options[:contigs_file]) do |seq|
+      sequence_names.push seq.definition
+
+      sequence = seq.seq
+      0.step(sequence.length-1-options[:probe_length], options[:probe_spacing]) do |offset|
+        # Only probe in the forward direction
+        probe_sequence = sequence[offset...offset+options[:probe_length]]
+        probe_sequences.push probe_sequence
+      end
+    end
+    log.info "Searching from #{probe_sequences.length} different probes from #{sequence_names.length} contigs)"
+
+    # Generate the graph with the probe sequences in it.
+    read_input = Bio::FinishM::ReadInput.new
+    read_input.parse_options options
+    finishm_graph = Bio::FinishM::GraphGenerator.new.generate_graph(probe_sequences, read_input, options)
+
+    # Loop over the ends, trying to make connections from each one
+    fluffer = Bio::AssemblyGraphAlgorithms::Fluffer.new
+    fluffings = fluffer.fluff(finishm_graph, options[:graph_search_leash_length])
+    log.debug "Found these fluffings: #{fluffings}" if log.debug?
+    log.info "Found #{fluffings.collect{|sets| sets.length}.reduce(:+)} paths in total" if log.info?
+
+    if options[:output_graph_png] or options[:output_graph_svg] or options[:output_graph_dot]
+      log.info "Converting assembly to a graphviz PNG"
+      viser = Bio::Assembly::ABVisualiser.new
+      gv = viser.graphviz(finishm_graph.graph, {:start_node_ids => finishm_graph.probe_nodes.collect{|node| node.node_id}})
+
+      gv.output :png => options[:output_graph_png], :use => :neato if options[:output_graph_png]
+      gv.output :svg => options[:output_graph_svg], :use => :neato if options[:output_graph_svg]
+      gv.output :dot => options[:output_graph_dot] if options[:output_graph_dot]
+    end
+
+    # Print out the sequences
+    File.open(options[:output_fluff_file], 'w') do |output|
+      fluffings.each_with_index do |path_set, probe_number|
+        path_set.each_with_index do |path, path_number|
+          fate = path_set.fates[path_number]
+          output.puts ">probe#{probe_number+1}_path#{path_number+1} #{fate}"
+          output.puts path.sequence
+        end
+      end
+    end
+  end
+end

data/lib/finishm/gapfiller.rb

@@ -0,0 +1,325 @@
+require 'tmpdir'
+
+class Bio::FinishM::GapFiller
+  include Bio::FinishM::Logging
+
+  def add_options(optparse_object, options)
+    optparse_object.banner = "\nUsage: finishm gapfill --contigs <contigs_file> --fastq-gz <reads..> --output-fasta <output.fa>
+
+Takes a set of reads and a contig that contains gap characters. Then it tries to fill in
+these N characters. It is possible that there is multiple ways to close the gap - in that case
+each can be reported.
+
+example: finishm gapfill --contigs to_gapfill.fasta --fastq-gz reads.1.fq.gz,reads.2.fq.gz --output-fasta output.fasta
+\n"
+
+    options.merge!({
+      :contig_end_length => 200,
+      :graph_search_leash_length => 20000,
+      })
+
+    optparse_object.separator "\nRequired arguments:\n\n"
+    optparse_object.on("--contigs FILE", "fasta file of single contig containing Ns that are to be closed [required]") do |arg|
+      options[:contigs_file] = arg
+    end
+    optparse_object.on("--output-fasta PATH", "Output the gap-filled sequence to this file [required]") do |arg|
+      options[:overall_fasta_file] = arg
+    end
+
+    optparse_object.separator "\nThere must be some definition of of how to do the assembly, or else a path to a previous assembly directory:\n\n"
+    Bio::FinishM::ReadInput.new.add_options(optparse_object, options)
+    Bio::FinishM::GraphGenerator.new.add_options optparse_object, options
+
+    optparse_object.separator "\nGraph search options:\n\n"
+    optparse_object.on("--overhang NUM", Integer, "Start assembling this many base pairs back from the gap [default: #{options[:contig_end_length] }]") do |arg|
+      options[:contig_end_length] = arg
+    end
+    optparse_object.on("--leash-length NUM", Integer, "Don't explore too far in the graph, only this many base pairs and not (much) more [default: #{options[:graph_search_leash_length] }]") do |arg|
+      options[:graph_search_leash_length] = arg
+    end
+    optparse_object.on("--recoherence-kmer NUM", Integer, "Use a kmer longer than the original velvet one, to help remove bubbles and circular paths [default: none]") do |arg|
+      options[:recoherence_kmer] = arg
+    end
+
+    optparse_object.separator "\nVisualisation options (of all joins):\n\n"
+    optparse_object.on("--assembly-png PATH", "Output assembly as a PNG file [default: off]") do |arg|
+      options[:output_graph_png] = arg
+    end
+    optparse_object.on("--assembly-svg PATH", "Output assembly as an SVG file [default: off]") do |arg|
+      options[:output_graph_svg] = arg
+    end
+    optparse_object.on("--assembly-dot PATH", "Output assembly as an DOT file [default: off]") do |arg|
+      options[:output_graph_dot] = arg
+    end
+  end
+
+  def validate_options(options, argv)
+    #TODO: give a better description of the error that has occurred
+    #TODO: require reads options
+    if argv.length != 0
+      return "Dangling argument(s) found e.g. #{argv[0] }"
+    else
+      [
+        :contigs_file,
+        :overall_fasta_file
+        ].each do |sym|
+          if options[sym].nil?
+            return "No option found to specify #{sym}"
+          end
+        end
+
+      #if return nil from here, options all were parsed successfully
+      return Bio::FinishM::ReadInput.new.validate_options(options, [])
+    end
+  end
+
+  def run(options, argv)
+    # Read in all the contigs sequences and work out where the gaps are
+    genome = Bio::FinishM::InputGenome.new(
+      options[:contigs_file],
+      options[:contig_end_length],
+      options
+      )
+
+
+    scaffolds = Bio::FinishM::ScaffoldBreaker.new.break_scaffolds(options[:contigs_file])
+    gaps = []
+    output_fasta_file = File.open(options[:overall_fasta_file],'w')
+    num_without_gaps = 0
+    scaffolds.each do |scaffold|
+      sgaps = scaffold.gaps
+      if sgaps.empty?
+        num_without_gaps += 1
+        output_fasta_file.puts ">#{scaffold.name }"
+        output_fasta_file.puts scaffold.sequence
+      else
+        gaps.push scaffold.gaps
+      end
+    end
+    gaps.flatten!
+    log.info "Detected #{gaps.length} gap(s) from #{scaffolds.length} different contig(s). #{num_without_gaps } contig(s) were gap-free."
+
+    # Create probe sequences
+    probe_sequences = []
+    gaps.each do |gap|
+      sequence = gap.scaffold.sequence
+
+      if gap.start < options[:contig_end_length] or gap.stop > sequence.length - options[:contig_end_length]
+        log.warn "Found a gap that was too close to the end of a contig, skipping it: #{gap.coords}"
+        next
+      end
+
+      log.debug "Processing gap number #{gap.number}, #{gap.coords}"
+      first_coords = [
+        gap.start-options[:contig_end_length]-1,
+        gap.start-1,
+        ]
+      second_coords = [
+        gap.stop,
+        (gap.stop+options[:contig_end_length]),
+        ]
+      log.debug "Coordinates of the probes are #{first_coords} and #{second_coords}"
+      second = sequence[second_coords[0]..second_coords[1]]
+      probes = [
+        sequence[first_coords[0]...first_coords[1]],
+        Bio::Sequence::NA.new(second).reverse_complement.to_s,
+        ]
+      #TODO: this could probably be handled better.. e.g. if the amount of sequence is too small, just throw it out and make one big gap
+      if probes[0].match(/N/i) or probes[1].match(/N/i)
+        log.warn "Noticed gap that was too close together, skipping: #{gap.coords}"
+        next
+      end
+      probe_sequences.push probes[0]
+      probe_sequences.push probes[1]
+    end
+    log.debug "Generated #{probe_sequences.length} probes e.g. #{probe_sequences[0] }"
+
+
+    # Generate the graph with the probe sequences in it.
+    read_input = Bio::FinishM::ReadInput.new
+    read_input.parse_options options
+    # Own the tmpdir, if one is to be used - need to re-read the LastGraph later on see..
+    assembly_directory = options[:output_assembly_path]
+    assembly_directory ||= options[:previous_assembly]
+    using_tmp_assembly_directory = false
+    if assembly_directory.nil?
+      using_tmp_assembly_directory = true
+      assembly_directory = Dir.mktmpdir
+      options[:output_assembly_path] = assembly_directory
+    end
+
+    # Do the actual graph building and/or initial reading
+    options[:parse_sequences] = true
+    finishm_graph = Bio::FinishM::GraphGenerator.new.generate_graph(probe_sequences, read_input, options)
+
+    # Output optional graphics.
+    if options[:output_graph_png] or options[:output_graph_svg] or options[:output_graph_dot]
+      viser = Bio::Assembly::ABVisualiser.new
+      # TODO: make these visualise more than one join somehow
+      gv = viser.graphviz(finishm_graph.graph, {
+        :start_node_id => finishm_graph.probe_nodes[0].node_id,
+        :end_node_id => finishm_graph.probe_nodes[1].node_id})
+
+      if options[:output_graph_png]
+        log.info "Converting assembly to a graphviz PNG"
+        gv.output :png => options[:output_graph_png], :use => :neato
+      end
+      if options[:output_graph_svg]
+        log.info "Converting assembly to a graphviz SVG"
+        gv.output :svg => options[:output_graph_svg], :use => :neato
+      end
+      if options[:output_graph_dot]
+        log.info "Converting assembly to a graphviz DOT"
+        gv.output :dot => options[:output_graph_dot]
+      end
+    end
+
+    # Clean up the tmdir, if one was used.
+    if using_tmp_assembly_directory
+      log.debug "Removing tmpdir that held the assembly `#{assembly_directory}'.."
+      FileUtils.remove_entry assembly_directory
+    end
+
+    # Do the gap-filling and print out the results
+    printer = Bio::AssemblyGraphAlgorithms::ContigPrinter.new
+    num_total_trails = 0
+    num_singly_filled = 0
+    num_unbridgable = 0
+
+    output_trails_file = nil
+    output_trails_file = File.open(options[:overall_trail_output_fasta_file],'w') unless options[:overall_trail_output_fasta_file].nil?
+
+    # Print the fasta output for the scaffold
+    print_scaffold = lambda do |last_scaffold, gapfilled_sequence|
+      output_fasta_file.puts ">#{last_scaffold.name }"
+      #gapfilled_sequence += last_scaffold.contigs[last_scaffold.contigs.length-1].sequence #add last contig
+      output_fasta_file.puts gapfilled_sequence
+    end
+    # Lambda to add a gap the the String representing the scaffold
+    #TODO: if the trail is not filled then the wrong sequence is currently printed. BUG???
+    filler = lambda do |anchored_connection, following_contig, gapfilled_sequence, gap|
+      gapfilled = nil
+      if anchored_connection.paths.length == 1
+        # If there is only 1 trail, then output scaffolding information
+        num_singly_filled += 1
+
+        gapfilled = printer.one_connection_between_two_contigs(
+          finishm_graph.graph,
+          gapfilled_sequence,
+          anchored_connection,
+          following_contig.sequence
+          )
+      else
+        # Otherwise don't make any assumptions
+        num_unbridgable += 1 if anchored_connection.paths.empty?
+        # TODO: even the there is multiple trails, better info can still be output here
+        gapfilled = gapfilled_sequence + 'N'*gap.length + following_contig.sequence
+      end
+      gapfilled #return this string
+    end
+
+    log.info "Searching for trails between the nodes within the assembly graph"
+    log.info "Using contig overhang length #{options[:contig_end_length] } and leash length #{options[:graph_search_leash_length] }"
+    gapfilled_sequence = ''
+    last_scaffold = nil
+
+    (0...(probe_sequences.length / 2)).collect{|i| i*2}.each do |start_probe_index|
+      gap_number = start_probe_index / 2
+      gap = gaps[gap_number]
+      log.info "Now working through gap number #{gap_number+1}: #{gap.coords}"
+
+      probe_index1 = start_probe_index
+      probe_index2 = start_probe_index+1
+
+      connection = gapfill(finishm_graph, probe_index1, probe_index2, options)
+      log.info "Found #{connection.paths.length} trails for #{gap.coords}"
+
+      unless output_trails_file.nil?
+        # print the sequences of the trails if asked for:
+        trails.each_with_index do |trail, i|
+          #TODO: need to output this as something more sensible e.g. VCF format
+          output_trails_file.puts ">#{gap.coords}_trail#{i+1}"
+          output_trails_file.puts trail.sequence
+        end
+      end
+      num_total_trails += connection.paths.length
+
+      # Output the updated sequence. Fill in the sequence if there is only 1 trail
+      if gap.scaffold == last_scaffold
+        # We are still building the current scaffold
+        #gapfilled_sequence += gap.scaffold.contigs[gap.number].sequence
+        log.debug "Before adding next chunk of contig, length of scaffold being built is #{gapfilled_sequence.length}" if log.debug?
+        gapfilled_sequence = filler.call connection, gap.scaffold.contigs[gap.number+1], gapfilled_sequence, gap
+        log.debug "After adding next chunk of contig, length of scaffold being built is #{gapfilled_sequence.length}" if log.debug?
+      else
+        # We are onto a new scaffold. Print the previous one (unless this the first one)
+        unless last_scaffold.nil?
+          # print the gapfilled (or not) scaffold.
+          print_scaffold.call(last_scaffold, gapfilled_sequence)
+        end
+        #reset
+        last_scaffold = gap.scaffold
+
+        #add the current gap (and the contig before it)
+        log.debug "Before adding first chunk of contig, length of scaffold being built is #{gapfilled_sequence.length}"
+        gapfilled_sequence = gap.scaffold.contigs[gap.number].sequence
+        log.debug "After adding first chunk of contig, length of scaffold being built is #{gapfilled_sequence.length}"
+        gapfilled_sequence = filler.call connection, gap.scaffold.contigs[gap.number+1], gapfilled_sequence, gap
+        log.debug "After adding first gap sequence and next contig, gapfilled sequence length is #{gapfilled_sequence.length}"
+      end
+    end
+    print_scaffold.call(last_scaffold, gapfilled_sequence) # print the last scaffold
+
+    log.info "#{num_unbridgable } gaps had no suitable bridging paths in the graph within the leash, and found #{num_total_trails} trails in total."
+    log.info "Filled #{num_singly_filled } out of #{gaps.length } gaps."
+
+    output_trails_file.close unless output_trails_file.nil?
+    output_fasta_file.close
+  end
+
+  # Given a finishm graph, gapfill from the first probe to the second. Return a
+  # Bio::AssemblyGraphAlgorithms::ContigPrinter::AnchoredConnection object
+  def gapfill(finishm_graph, probe_index1, probe_index2, options)
+    start_onode = finishm_graph.velvet_oriented_node(probe_index1)
+    end_onode_inward = finishm_graph.velvet_oriented_node(probe_index2)
+    unless start_onode and end_onode_inward
+      raise "Unable to retrieve both probes from the graph for gap #{gap_number} (#{gap.coords}), fail"
+    end
+
+    # The probe from finishm_graph points in the wrong direction for path finding
+    end_onode = Bio::Velvet::Graph::OrientedNodeTrail::OrientedNode.new
+    end_onode.node = end_onode_inward.node
+    end_onode.first_side = end_onode_inward.starts_at_start? ? Bio::Velvet::Graph::OrientedNodeTrail::END_IS_FIRST : Bio::Velvet::Graph::OrientedNodeTrail::START_IS_FIRST
+
+    adjusted_leash_length = finishm_graph.adjusted_leash_length(probe_index1, options[:graph_search_leash_length])
+    log.debug "Using adjusted leash length #{adjusted_leash_length }" if log.debug?
+
+    cartographer = Bio::AssemblyGraphAlgorithms::AcyclicConnectionFinder.new
+    trails = cartographer.find_trails_between_nodes(
+      finishm_graph.graph, start_onode, end_onode, adjusted_leash_length, {
+        :recoherence_kmer => options[:recoherence_kmer],
+        :sequences => finishm_graph.velvet_sequences,
+        :max_explore_nodes => options[:max_explore_nodes],
+        :max_gapfill_paths => options[:max_gapfill_paths],
+        }
+      )
+    if trails.circular_paths_detected
+      log.warn "Circular path detected here, not attempting to gapfill"
+    end
+    # Convert the trails into OrientedNodePaths
+    trails = trails.collect do |trail|
+      path = Bio::Velvet::Graph::OrientedNodeTrail.new
+      path.trail = trail
+      path
+    end
+
+    acon = Bio::AssemblyGraphAlgorithms::ContigPrinter::AnchoredConnection.new
+    acon.start_probe_noded_read = finishm_graph.probe_node_reads[probe_index1]
+    acon.end_probe_noded_read = finishm_graph.probe_node_reads[probe_index2]
+    acon.start_probe_contig_offset = options[:contig_end_length]
+    acon.end_probe_contig_offset = options[:contig_end_length]
+    acon.paths = trails
+
+    return acon
+  end
+end