finishm 0.0.1

data/bin/read_selection_by_kmer.d

@@ -0,0 +1,174 @@
+#!/usr/bin/env rdmd
+
+import std.stdio;
+import std.string;
+import std.conv;
+import std.getopt;
+import std.file;
+import std.array;
+import bio.core.fasta;
+import bio.core.sequence;
+import std.algorithm;
+import std.container;
+import std.c.stdlib;
+
+void main(string[] args){
+  string whitelistFile, blacklistFile, fastaFile = null;
+  bool help, verbose, quiet, debugging = false;
+  int targetPerKmer = 1;
+  int minLeftoverLength;
+  getopt(args,
+    "whitelist",  &whitelistFile,
+    "blacklist",  &blacklistFile,
+    "reads",  &fastaFile,
+    "kmer-coverage-target", &targetPerKmer,
+    "min-leftover-length", &minLeftoverLength,
+    "verbose", &verbose,
+    "debug", &debugging,
+    "quiet", &quiet,
+    "help|h",         &help,
+  );
+  if(help){
+    writeln("my helpa");
+  }
+  else if(whitelistFile is null){stderr.writeln("Error: Need to specify a newline-separeted list of whitelisted kmers as --whitelist <file>");}
+  else if(fastaFile is null){stderr.writeln("Error: Need to specify a fasta file of reads to work with as --reads <fasta_file>");}
+  else {
+    if(debugging) verbose = true;
+    if(verbose) quiet=false;
+
+
+    //read in a text file of kmers that we wish to find, the whitelist
+    auto whites = split(cast(string) read(whitelistFile));
+    if (verbose)
+      stderr.writeln("Read in ",whites.length," kmers as a whitelist, e.g. ",whites.front);
+
+
+    // Find the minimum length of kmer being searched for
+    auto whitelistMinLength = map!"a.length"(whites).reduce!"a<b ? a : b";
+    auto whitelistMaxLength = map!"a.length"(whites).reduce!"a<b ? b : a";
+    if (whitelistMinLength != whitelistMaxLength){
+      stderr.writeln("Kmers must be of uniform length, but these ones weren't..");
+      exit(1);
+    }
+    if (verbose)
+      stderr.writeln("Minimum length of kmer in whitelist is ",whitelistMinLength);
+
+    //if blacklistFile is specified, read in a list of kmers that are blacklisted, otherwise make an empty array
+    bool[string] blacks;
+    if(blacklistFile != null){
+      foreach(kmer; split(cast(string) read(blacklistFile))){
+        if (kmer.length != whitelistMinLength){
+          stderr.writeln("Kmers (currently) must be of uniform length, but some blacklisted ones weren't..");
+          exit(1);
+        }
+        blacks[kmer] = true;
+        if(verbose)
+          stderr.writeln("Read in ",blacks.length," blacklisted kmers e.g. ",blacks.keys.front);
+      }
+    } else {
+      if(verbose)
+        stderr.writeln("No blacklisted kmers specified");
+    }
+
+    int[string] whitelistCounts;
+    foreach(white; whites){
+      whitelistCounts[white] = 0;
+    }
+    int num_reads_whitelisted = 0;
+    int num_reads_blacklisted = 0;
+
+    //Iterate through the fastq reads given.
+    auto fastas = fastaRecords(fastaFile);
+    bool all_accounted_for = false;
+    ptrdiff_t range_end;
+    string[] kmers;
+    if (minLeftoverLength)
+      kmers = new string[4];
+    else
+      kmers = new string[2];
+    foreach(seq; fastas){
+      if (verbose)
+        stderr.writeln("Inspecting ", seq);
+      //If they contain one of the blacklist kmers, then skip
+      string fwd = seq.sequence;
+      string rev = to!string(nucleotideSequence(fwd, true));
+
+      range_end = fwd.length - whitelistMinLength + 1;
+      if (minLeftoverLength)
+        range_end -= minLeftoverLength;
+      if (range_end < 0) continue; //If the read is too short, then don't even bother comparing it
+      if (debugging) stderr.writeln("Range end was ",range_end);
+
+      //How many of each whitelist kmers are found (including in the reverse complement)?
+      bool whitelisted = false;
+      foreach(i; 0 .. range_end){
+        kmers[0] = fwd[i .. (i+whitelistMinLength)];
+        kmers[1] = rev[i .. (i+whitelistMinLength)];
+        // if min leftover length is specified then search the reverse complement of the fwd as well
+        if (minLeftoverLength){
+          kmers[2] = to!string(nucleotideSequence(kmers[0], true));
+          kmers[3] = to!string(nucleotideSequence(kmers[1], true));
+        }
+        foreach(kmer; kmers){
+          if (debugging)
+            stderr.writeln("Whitelist inspecting kmer ",kmer," at position ",i);
+          if (kmer in whitelistCounts && whitelistCounts[kmer] < targetPerKmer){
+            whitelisted = true;
+            whitelistCounts[kmer] += 1;
+            if (whitelistCounts[kmer] >= targetPerKmer){
+              if(verbose)
+                stderr.writeln("kmer index ",i," now accounted for");
+              if (count!((x){return x<targetPerKmer;})(whitelistCounts.values) == 0){
+                if(verbose)
+                  stderr.writeln("All whitelisted kmers now accounted for");
+                all_accounted_for = true; //all done, no more fasta entries required
+              }
+            }
+          }
+        }
+      }
+      if(!whitelisted) continue;
+      else if (verbose) stderr.writeln("Read contains a valid whitelisted kmer");
+
+      //I'm sure there is a faster way to search for an array of strings within a particular string, but eh for now.
+      bool blacklisted = false;
+      if (blacklistFile != null){
+        foreach(i; 0 .. fwd.length - whitelistMinLength+1){
+          auto kmer = fwd[i .. (i+whitelistMinLength)];
+          if (kmer in blacks){
+            //blacklisted kmer found
+            blacklisted = true;
+            break;
+          }
+        }
+      }
+
+      if(blacklisted){
+        num_reads_blacklisted += 1;
+        if(verbose)
+          stderr.writeln(fwd," contains a blacklisted kmer, not including this one");
+        continue;
+      } else {
+        if(verbose)
+          stderr.writeln(fwd," not blacklisted");
+      }
+
+      //print this sequence, as it is whitelisted and not blacklisted
+      num_reads_whitelisted += 1;
+      writeln(">", seq.header);
+      writeln(fwd);
+
+      if(all_accounted_for) break;
+    }
+
+    //output the number of kmers that were sufficiently covered
+
+    ulong num_counted = count!("a >= b")(whitelistCounts.values, targetPerKmer);
+    ulong num_not_counted = whitelistCounts.length - num_counted;
+    if(!quiet){
+      stderr.writeln("Found ",num_counted," from the whitelist as expected and ",num_not_counted," not enough times");
+      stderr.writeln("There were ",num_reads_whitelisted," reads output, and ",num_reads_blacklisted," reads blacklisted");
+    }
+  }
+}

data/bin/scaffold_by_pattern.rb

@@ -0,0 +1,119 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'bio-logger'
+require 'csv'
+require 'bio'
+require 'tempfile'
+require 'systemu'
+
+SCRIPT_NAME = File.basename(__FILE__); LOG_NAME = SCRIPT_NAME.gsub('.rb','')
+$:.unshift File.join(File.dirname(__FILE__),'..','lib')
+require 'kmer_abundance_pattern'
+
+# Parse command line options into the options hash
+options = {
+  :logger => 'stderr',
+  :log_level => 'info',
+  :number_of_kmers => 100,
+}
+o = OptionParser.new do |opts|
+  opts.banner = "
+    Usage: #{SCRIPT_NAME} -f <scaffolds_fasta> -k <kmer_abundance_file>
+
+    Take a fasta file of contigs, and a multiple kmer count file. Output the patterns each contig end shows up.n\n"
+
+
+  opts.on("-f FASTA_FILE", "Fasta file containing multiple sequences that we are attempting to scaffold together [required]") do |arg|
+    options[:fasta_file] = arg
+  end
+  opts.on("-k KMER_FILE", "kmer frequencies [required]") do |arg|
+    options[:kmer_file] = arg
+  end
+  opts.on("--kmer KMER_SIZE", "kmer length [required]") do |arg|
+    options[:kmer_size] = arg.to_i
+  end
+  opts.on("--upper-threshold ARG", "kmer frequency cutoff to saying 'present' [required]") do |arg|
+    options[:upper_threshold] = arg.to_i
+  end
+  opts.on("--lower-threshold ARG", "kmer frequency cutoff to saying 'not present' [required]") do |arg|
+    options[:lower_threshold] = arg.to_i
+  end
+
+  # logger options
+  opts.separator "\nVerbosity:\n\n"
+  opts.on("-q", "--quiet", "Run quietly, set logging to ERROR level [default INFO]") {options[:log_level] = 'error'}
+  opts.on("--logger filename",String,"Log to file [default #{options[:logger]}]") { |name| options[:logger] = name}
+  opts.on("--trace options",String,"Set log level [default INFO]. e.g. '--trace debug' to set logging level to DEBUG"){|s| options[:log_level] = s}
+end; o.parse!
+if ARGV.length != 0 or options[:fasta_file].nil? or options[:kmer_file].nil? or options[:upper_threshold].nil? or options[:lower_threshold].nil? or options[:kmer_size].nil?
+  $stderr.puts o
+  exit 1
+end
+# Setup logging
+Bio::Log::CLI.logger(options[:logger]); Bio::Log::CLI.trace(options[:log_level]); log = Bio::Log::LoggerPlus.new(LOG_NAME); Bio::Log::CLI.configure(LOG_NAME)
+
+get_kmer_abundances_from_kmers = lambda do |kmers|
+  # fgrep the kmer abundance file for these particular ones
+  patterns = {}
+  Tempfile.open('kemrs') do |tempfile|
+    tempfile.puts kmers.join "\n"
+    tempfile.close
+
+    # for each of the kmers that come back, output their pattern in the kmer abundance file
+    grep_cmd = "fgrep -f #{tempfile.path} #{options[:kmer_file].inspect}"
+    log.debug "Running cmd with #{kmers.length} kmers: #{grep_cmd}"
+    status, stdout, stderr = systemu grep_cmd
+    raise stderr if stderr != ''
+    raise unless status.exitstatus == 0
+    num_kmers = stdout.split("\n").length
+    log.debug "Finished grepping for kmers, found #{num_kmers} kmers"
+
+
+
+    stdout.each_line do |line|
+      CSV.parse(line, :col_sep => ' ') do |row|
+        pattern = KmerAbundancePattern.new
+        pattern.parse_from_kmer_abundance row[1...row.length].collect{|a| a.to_f}, options[:lower_threshold], options[:upper_threshold]
+        rep = pattern.binary_string
+        patterns[rep] ||= 0
+        patterns[rep] += 1
+      end
+    end
+  end
+  patterns.sort{|a,b| b[1]<=>a[1]}.collect{|a| a.join(',')}.join("\t")
+end
+
+# For each of the sequences in the fasta file
+Bio::FlatFile.foreach(Bio::FastaFormat, options[:fasta_file]) do |seq|
+  # Extract the first 100 kmers
+  kmers = []
+  i = 0
+  seq.to_biosequence.window_search(options[:kmer_size]) do |s|
+    kmers.push s.seq
+    i += 1
+    break if i>options[:number_of_kmers]
+  end
+  # output to a tempfile
+  patterns = get_kmer_abundances_from_kmers.call kmers
+  puts [
+    seq.definition,
+    'start',
+    patterns
+  ].join("\t")
+
+  # repeat for the end of the contig
+  kmers = []
+  i = 0
+  seq.naseq.reverse_complement.window_search(options[:kmer_size]) do |s|
+    kmers.push s.seq.upcase
+    i += 1
+    break if i>options[:number_of_kmers]
+  end
+  patterns = get_kmer_abundances_from_kmers.call kmers
+  puts [
+    seq.definition,
+    'end',
+    patterns
+  ].join("\t")
+end

data/bin/scaffold_connection_possibilities_to_knowns.rb

@@ -0,0 +1,193 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'bio-logger'
+require 'csv'
+require 'pp'
+require 'bio'
+require 'pry'
+
+SCRIPT_NAME = File.basename(__FILE__); LOG_NAME = SCRIPT_NAME.gsub('.rb','')
+
+$:.unshift File.join(ENV['HOME'],'git/yargraph/lib')
+require 'yargraph'
+
+# Parse command line options into the options hash
+options = {
+  :logger => 'stderr',
+  :log_level => 'info',
+}
+o = OptionParser.new do |opts|
+  opts.banner = "
+    Usage: #{SCRIPT_NAME} <arguments>
+
+    Description of what this program does...\n\n"
+
+  opts.on("-c", "--connnections FILE", "connections file [required]") do |arg|
+    options[:connections_file] = arg
+  end
+  opts.on("-f", "--fasta FILE", "Fasta file of all contigs [required]") do |arg|
+    options[:fasta_file] = arg
+  end
+
+  # logger options
+  opts.separator "\nVerbosity:\n\n"
+  opts.on("-q", "--quiet", "Run quietly, set logging to ERROR level [default INFO]") {options[:log_level] = 'error'}
+  opts.on("--logger filename",String,"Log to file [default #{options[:logger]}]") { |name| options[:logger] = name}
+  opts.on("--trace options",String,"Set log level [default INFO]. e.g. '--trace debug' to set logging level to DEBUG"){|s| options[:log_level] = s}
+end; o.parse!
+if ARGV.length != 0 or options[:connections_file].nil?
+  $stderr.puts o
+  exit 1
+end
+# Setup logging
+Bio::Log::CLI.logger(options[:logger]); Bio::Log::CLI.trace(options[:log_level]); log = Bio::Log::LoggerPlus.new(LOG_NAME); Bio::Log::CLI.configure(LOG_NAME)
+
+class Probe
+  attr_accessor :contig_name, :side
+
+  def to_setable
+    [@contig_name, @side]
+  end
+end
+
+class ProbeSet
+
+end
+
+graph = Yargraph::UndirectedGraph.new
+
+num_probes_circular = 0
+CSV.foreach(options[:connections_file], :col_sep => "\t") do |row|
+  if row.length == 3
+    # e.g. 110811_E_1_D_nesoni_single_contig_1011407.start	110811_E_1_D_nesoni_single_contig_1030181.start	225
+    splits1 = nil
+    splits2 = nil
+    splits1 = row[0].match(/^(.+)\.(.+)$/)
+    splits2 = row[1].match(/^(.+)\.(.+)$/)
+    raise if splits1.nil? or splits2.nil?
+    distance = row[2]
+    row = [
+      splits1[1], splits1[2], splits2[1], splits2[2], distance
+      ].flatten
+  end
+
+  raise unless row.length == 5
+  # e.g. seq1    end     seq23   start   6878
+
+  probe1 = Probe.new
+  probe1.contig_name = row[0]
+  probe1.side = row[1]
+
+  probe2 = Probe.new
+  probe2.contig_name = row[2]
+  probe2.side = row[3]
+
+  if probe1.contig_name == probe2.contig_name and probe1.side == probe2.side
+    num_probes_circular += 1
+  else
+    graph.add_edge probe1.to_setable, probe2.to_setable
+  end
+end
+
+probes = graph.vertices.to_a
+
+# Connect all starts to the ends
+probes.each do |array|
+  contig_name = array[0]
+
+  start_probe = Probe.new
+  start_probe.contig_name = contig_name
+  start_probe.side = 'start'
+  end_probe = Probe.new
+  end_probe.contig_name = contig_name
+  end_probe.side = 'end'
+
+  graph.add_edge start_probe.to_setable, end_probe.to_setable
+end
+log.info "Removed #{num_probes_circular} connections that join a contig end to itself"
+
+# First try the not computationally intensive way - can we find any?
+edge_result = graph.some_edges_in_all_hamiltonian_cycles
+
+cross_contig_connections = []
+edge_result.edges_in_all.each do |v1, v2|
+  if v1[0] != v2[0]
+    cross_contig_connections.push [v1,v2]
+  end
+end
+
+if !cross_contig_connections.empty?
+  length = cross_contig_connections.length
+
+  log.info "Good news. Found #{length} connections that are in all Hamiltonian paths (and thus can probably be scaffolded together):"
+  cross_contig_connections.each do |connection|
+    log.info connection[0].to_s + "\t" + connection[1].to_s
+  end
+  if length == probes.length and edge_result.contains_hamcycle != false
+    log.info "Extra good news. You just scaffolded your genome into a single scaffold"
+  end
+end
+if edge_result.contains_hamcycle == false
+  log.warn "Bad news. The connectivity graph contains no Hamiltonian cycles, and so the contigs cannot be scaffolded into one circular genome"
+end
+
+# Determine if there are any ends that don't connect to anything
+contig_names = []
+Bio::FlatFile.foreach(options[:fasta_file]) do |seq|
+  contig_name = seq.definition.split(/\s+/)[0]
+  contig_names.push contig_name
+  %w(start end).each do |side|
+    probe = Probe.new
+    probe.contig_name = contig_name
+    probe.side = side
+    if graph.edges[probe.to_setable].empty?
+      log.info "Unable to find any connections from #{probe.to_setable}"
+    end
+  end
+end
+
+# Determine if there is any possible plasmids
+num_plasmids_found = 0
+contig_names.each do |contig_name|
+  probe = Probe.new
+  probe.contig_name = contig_name
+  probe.side = 'start'
+  rev_probe = Probe.new
+  rev_probe.contig_name = contig_name
+  rev_probe.side = 'end'
+
+  # Both the start and the end must only connect to each other
+  if graph.edges[probe.to_setable].length == 1 and
+    graph.edges[rev_probe.to_setable].length == 1 and
+    graph.edges[probe.to_setable].include?(rev_probe.to_setable)
+
+    num_plasmids_found += 1
+    log.info "Contig #{contig_name} appears to be circular and not connect to other contigs, suggesting it may be a plasmid"
+  end
+end
+log.info "Found #{num_plasmids_found} contigs that appear to be plasmids based on connectivity"
+
+
+
+log.info "Attempting a better but more computationally intensive method of determining edges that are in all hamiltonian paths.."
+# First try to see if there is any hamiltonian paths?
+paths = []
+max_path_count = 4
+operation_limit = 50000
+graph.hamiltonian_cycles(operation_limit) do |path|
+  if paths.length <= max_path_count
+    paths.push path
+  else
+    break
+  end
+end
+
+if paths.length < max_path_count
+  log.info "Found exactly #{paths.length} Hamiltonian cycles"
+else
+  log.info "Gave up searching for Hamiltonian cycles as there are at least #{max_path_count} cycles"
+end
+
+# OK so
+#edges_in_all = graph.edges_in_all_hamiltonian_cycles