big_simon 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 544705d81f538f636b61b014d4fa84d0a0b68918
-  data.tar.gz: e8943460d42bca98943fc27ae7919601cefcbe1a
+  metadata.gz: dfbea3a58014cbec45a3959ad076fd842c6b393c
+  data.tar.gz: 69d9f8ac1dea196f64dd13fb06a13a4c26b38e45
 SHA512:
-  metadata.gz: 8e0c2f5be8f1dbcdaa0149788db1fe05c2cdc7e5c0988d20311f8cf4a37aa3c0869d271704f606a00cab7d7bfb03e81a07f89bce52b92b933495fc62efdd26ac
-  data.tar.gz: e50a57bb282bb20274cb49673b657574b277d61af80c0a510caa6938fc5699eb368d3bcc133d9759d53dab1f6f57c3bd11c9c537227444bf6b6f7602b6f59bb4
+  metadata.gz: 3a54fe903bb5c0f2f574a389dfc945ffe3d93bd2b1a15361bda81430542c2ef3990ac31e67b671690e056e23544e4f30f21dbecd756050dfb98dfb924fd15ca3
+  data.tar.gz: b61f6ba10b7efc267419ccccaef82654662923629e0b2b61e5a3c5e74b79fe8b753dd160c3e3d7fd57b1e926b54684f9367a839075544c496ff3a839ece57fb5

data/.gitignore

@@ -18,4 +18,16 @@
 
 vendor/repos
 
-test_files/test_output
+test_files/test_output
+
+0000TEST
+
+apple.txt
+stats.r
+toyexample_out/
+
+spec/test_files/outdir_for_heatmaps/outdir/
+
+stats.tmp.r
+
+scratch

data/Gemfile.lock

@@ -1,14 +1,17 @@
 PATH
   remote: .
   specs:
-    big_simon (0.0.1)
-      rya (~> 0.1.3)
+    big_simon (0.1.0)
+      parse_fasta (~> 2.5, >= 2.5.2)
+      rya (~> 0.4.0)
+      trollop (~> 2.1, >= 2.1.3)
 
 GEM
   remote: https://rubygems.org/
   specs:
     abort_if (0.2.0)
     diff-lcs (1.3)
+    parse_fasta (2.5.2)
     rake (10.5.0)
     rspec (3.7.0)
       rspec-core (~> 3.7.0)
@@ -23,10 +26,11 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.7.0)
     rspec-support (3.7.1)
-    rya (0.1.3)
+    rya (0.4.0)
       abort_if (~> 0.2.0)
       systemu (~> 2.6, >= 2.6.5)
     systemu (2.6.5)
+    trollop (2.1.3)
 
 PLATFORMS
   ruby

data/Makefile

@@ -0,0 +1,8 @@
+test_small:
+	rm -r 0000TEST/; exe/big_simon -v spec/test_files/virus/* -h spec/test_files/host/* -o 0000TEST && tree 0000TEST
+
+test_small_install:
+	rm -r 0000TEST/; rake install && exe/big_simon -v spec/test_files/virus/* -h spec/test_files/host/* -o 0000TEST && tree 0000TEST
+
+test_toy:
+	rm -r toyexample_out; time exe/big_simon -v vendor/repos/VirHostMatcher/test/toyexample/virus/* -h vendor/repos/VirHostMatcher/test/toyexample/host/* -o toyexample_out -t 3

data/README.md

@@ -1,6 +1,8 @@
 # BigSimon
 
-A pipeline for finding hosts of viruses!
+Hi, I'm BigSimon (but you can call me BigSi if you want), and I'm a pipeline for finding hosts of viruses!
+
+Mainly, I'm just a wrapper for some other nice tools.
 
 ## Installation
 
@@ -20,7 +22,11 @@ Or install it yourself as:
 
 ## Usage
 
-TODO
+For now, there is not much documentation.  To see the help file, run:
+
+```
+$ big_simon --help
+```
 
 ## Development
 

data/big_simon.gemspec

@@ -26,5 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
 
-  spec.add_runtime_dependency "rya", "~> 0.1.3"
+  spec.add_runtime_dependency "parse_fasta", "~> 2.5", ">= 2.5.2"
+  spec.add_runtime_dependency "rya", "~> 0.4.0"
+  spec.add_runtime_dependency "trollop", "~> 2.1", ">= 2.1.3"
 end

data/exe/big_simon

@@ -0,0 +1,244 @@
+#!/usr/bin/env ruby
+Signal.trap("PIPE", "EXIT")
+
+require "pp"
+require "tempfile"
+
+require "parse_fasta"
+require "trollop"
+
+require "big_simon"
+
+# TODO make scaled scores with high score being better.
+
+Process.extend Rya::CoreExtensions::Process
+
+opts = Trollop.options do
+  version BigSimon::VERSION_BANNER
+
+  banner <<-EOS
+
+#{BigSimon::VERSION_BANNER}
+
+  Hi, I'm BigSimon!  I'm here to help you figure out the hosts for
+  your viruses.
+
+  I run a bunch of different programs.  In addition to doing some
+  merging of results, I'll give you heatmaps for all the programs and
+  you can check for yourself.
+
+  The scaled scores run from 0 to 1 with lower scores being better.
+
+  Options:
+  EOS
+
+  opt :viruses, "Path to fasta file(s) with viruses", type: :strings
+  opt :hosts, "Path to fasta file(s) with hosts", type: :strings
+  opt :outdir, "Output directory", default: "big_simon"
+  opt :threads, "Number of threads to use", default: 1
+end
+
+Rya::AbortIf.logger.debug { "Command line opts: #{opts.inspect}" }
+
+BigSimon::Utils.check_opt! opts, :viruses
+BigSimon::Utils.check_opt! opts, :hosts
+
+# Check infiles
+[opts[:viruses], opts[:hosts]].flatten.each do |fname|
+  BigSimon::Utils.check_file! fname
+end
+
+Rya::AbortIf.abort_unless opts[:threads] > 0,
+                          "--threads must be > 0"
+
+programs = [
+  "WIsH",
+  "VirHostMatcher",
+  "mummer"
+]
+
+outdir       = opts[:outdir]
+threads      = opts[:threads]
+virus_fnames = opts[:viruses]
+host_fnames  = opts[:hosts]
+
+FileUtils.mkdir_p outdir
+
+tmpdir       = File.join opts[:outdir], "big_simon_tmp"
+tmpdir_virus = File.join tmpdir, "virus"
+tmpdir_host  = File.join tmpdir, "host"
+
+# all_predictions_fname = File.join outdir, "scores_all.txt"
+mean_scaled_scores_fname = File.join outdir, "scores_scaled.mean.txt"
+
+virus_recs, host_recs = [], []
+
+# Tempfile.open do |vir_f|
+#   Tempfile.open do |host_f|
+#     virus_fnames.each do |fname|
+#       ParseFasta::SeqFile.open(fname).each_record do |rec|
+#         vir_f.puts rec
+#
+#         vir_f.puts ">#{rec.id}___reverse\n#{rec.seq.reverse}"
+#       end
+#     end
+#
+#     host_fnames.each do |fname|
+#       ParseFasta::SeqFile.open(fname).each_record do |rec|
+#         host_f.puts rec
+#         host_f.puts ">#{rec.id}___reverse\n#{rec.seq.reverse}"
+#       end
+#     end
+#
+#     vir_f.fsync
+#     host_f.fsync
+#
+#     cmd = "mummer -maxmatch -l 15 #{host_f.path} #{vir_f.path} > /Users/moorer/Desktop/mummer.OUT"
+#     Process.run_and_time_it! "MUMMER", cmd
+#   end
+# end
+#
+# header   = nil
+# hits     = []
+# hit_info = {}
+# virus = nil
+#
+# File.open("/Users/moorer/Desktop/mummer.OUT", "rt").each_line.with_index do |line, idx|
+#   if line.start_with? '>'
+#     virus = line.chomp.sub(/^>/, "").sub(/___reverse$/, "").strip
+#
+#     unless hit_info.has_key? virus
+#       hit_info[virus] = {}
+#     end
+#   else
+#     host, _, _, len = line.chomp.strip.split(" ")
+#     host = host.sub(/___reverse$/, "").strip
+#
+#     unless hit_info[virus].has_key? host
+#       hit_info[virus][host] = -1
+#     end
+#
+#     hit_info[virus][host] = len.to_i if len.to_i > hit_info[virus][host]
+#   end
+# end
+#
+# puts
+#
+# hh = hit_info.map do |virus, info|
+#   [virus, info.to_a.sort_by {|gen, len| len}.reverse]
+# end
+#
+# pp hh
+
+# hh = hit_info.map do |virus, info|
+#   [virus, info.to_a.sort_by { |host, hit_len| hit_len }.reverse
+#
+# end
+# p hit_info
+
+scores_files = {}
+programs.each do |program|
+  raw_fname    = File.join outdir, "scores_raw.#{program}.txt"
+  scaled_fname = File.join outdir, "scores_scaled.#{program}.txt"
+
+  scores_files[program] = {
+    raw:    File.open(raw_fname, "w"),
+    scaled: File.open(scaled_fname, "w")
+  }
+end
+
+scores_files.each do |program, files|
+  files.each do |name, file|
+    file.puts %w[virus host score].join "\t"
+  end
+end
+
+name_map_virus, all_ids_virus = BigSimon::Utils.set_up_tmp_dirs virus_fnames, tmpdir_virus, "virus"
+name_map_host, all_ids_host   = BigSimon::Utils.set_up_tmp_dirs host_fnames, tmpdir_host, "host"
+
+wish_outf = BigSimon::Runners.wish BigSimon::WISH, tmpdir_virus, tmpdir_host, tmpdir, threads
+vhm_outf  = BigSimon::Runners.vir_host_matcher BigSimon::VHM, tmpdir_virus, tmpdir_host, tmpdir
+
+host_info_mummer = BigSimon::Runners.mummer BigSimon::MUMMER, tmpdir_virus, tmpdir_host, tmpdir, threads
+
+# Map them back to simple names.  TODO just have it spit these out from the beginning.
+host_info_mummer_simple_names = {}
+inverted_name_map_virus = name_map_virus.invert
+inverted_name_map_host = name_map_host.invert
+
+host_info_mummer.each do |virus, host_tables|
+  virname = virus
+  if inverted_name_map_virus.has_key? virus
+    virname = inverted_name_map_virus[virus]
+  end
+
+  host_info_mummer_simple_names[virname] = []
+
+  host_tables.map do |table|
+    hostname = inverted_name_map_host.has_key?(table[:host]) ? inverted_name_map_host[table[:host]] : table[:host]
+    new_table = { host: hostname, score: table[:score], scaled_score: table[:scaled_score] }
+
+    host_info_mummer_simple_names[virname] << new_table
+  end
+end
+
+host_info_wish = BigSimon::Parsers.wish wish_outf
+host_info_vhm  = BigSimon::Parsers.vir_host_matcher vhm_outf
+
+host_info_simple_names = BigSimon::Pipeline.collate_host_results [host_info_wish, host_info_vhm, host_info_mummer_simple_names], programs
+host_info              = BigSimon::Pipeline.map_taxa host_info_simple_names, name_map_virus, name_map_host
+
+# puts
+# pp host_info
+# puts
+
+# Just a basic all info file
+# File.open all_predictions_fname, "w" do |f|
+#   f.puts %w[virus host program score scaled.score].join "\t"
+
+host_info.each do |virus, h1|
+  h1.each do |host, h2|
+    lines = {}
+
+    h2[:scores].each do |program, score|
+      lines[[virus, host, program]] = [score]
+
+      scores_files[program][:raw].puts [virus, host, score].join "\t"
+    end
+
+    # Add in the scaled score too.
+    h2[:scaled_scores].each do |program, score|
+      lines[[virus, host, program]] << score
+
+      scores_files[program][:scaled].puts [virus, host, score].join "\t"
+    end
+
+    # lines.each do |(virus, host, program), (score, scaled_score)|
+    #   f.puts [virus, host, program, score, scaled_score].join "\t"
+    # end
+  end
+end
+# end
+
+# A file with mean scaled scores.
+File.open mean_scaled_scores_fname, "w" do |f|
+  f.puts %w[virus host score].join "\t"
+
+  host_info.each do |virus, h1|
+    h1.each do |host, h2|
+      scaled_scores = h2[:scaled_scores].values
+
+      mean_scaled_score = scaled_scores.reduce(:+) / scaled_scores.length.to_f
+
+      f.puts [virus, host, mean_scaled_score].join "\t"
+    end
+  end
+end
+
+scores_files.each do |program, file|
+  file.values.map(&:close)
+end
+FileUtils.rm_r tmpdir
+
+# Make the heatmaps
+BigSimon::Runners.heatmaps BigSimon::RSCRIPT, outdir, File.join(outdir, "heatmaps")

data/exe/ranks

@@ -0,0 +1,93 @@
+#!/usr/bin/env ruby
+Signal.trap("PIPE", "EXIT")
+
+require "pp"
+
+by_program = {}
+lines = []
+
+TOP = (ENV["TOP"] || 5).to_i
+
+ARGV.each do |fname|
+  scores = {}
+  File.open(fname, "rt").each_line.with_index do |line, idx|
+    unless idx.zero?
+      virus, host, score = line.chomp.split "\t"
+
+      unless scores.has_key? virus
+        scores[virus] = []
+      end
+
+      scores[virus] << [host, score.to_f]
+    end
+  end
+
+  scores.sort_by { |virus, _| virus }.each do |virus, host_scores|
+    # Lowest score is the best
+    top_5 = host_scores.sort_by { |host, score| score }.take(TOP).map(&:first)
+
+    line = [File.basename(fname), virus, top_5]
+    lines << line
+
+    # puts line.join "\t"
+  end
+  # puts
+end
+
+lines.each do |line|
+  program, virus, all = line
+  first = all.first
+
+  unless by_program.has_key? virus
+    by_program[virus] = {}
+  end
+
+  unless program == "scores_scaled.mean.txt"
+    by_program[virus][program] = { first: first, all: all }
+  end
+end
+
+# These track the number of times a host shows up in the first spot and in the top N spots for that virus for all programs.
+first_table = {}
+top_host_table = {}
+
+by_program.each do |virus, program_tables|
+  first_table[virus] = {}
+  top_host_table[virus] = {}
+
+  program_tables.each do |program, top_info|
+
+
+    first_host = top_info[:first]
+    all_top = top_info[:all]
+
+    unless first_table[virus].has_key? first_host
+      first_table[virus][first_host] = []
+    end
+
+
+    first_table[virus][first_host] << program
+
+    all_top.each do |top_host|
+      unless top_host_table[virus].has_key? top_host
+        top_host_table[virus][top_host] = []
+      end
+
+      top_host_table[virus][top_host] << program
+    end
+  end
+end
+
+first_table.each do |virus, host_counts|
+  host_counts.sort_by { |_, programs| programs.count }.reverse.each do |host, programs|
+    STDERR.puts [virus, :best, host, programs.count, programs].join "\t"
+  end
+  STDERR.puts
+end
+
+top_host_table.each do |virus, host_counts|
+  host_counts.sort_by { |_, programs| programs.count }.reverse.each do |host, programs|
+      puts [virus, :top_N, host, programs.count, programs].join "\t"
+  end
+  puts
+end

data/lib/big_simon/parsers.rb

@@ -0,0 +1,78 @@
+module BigSimon
+  # Methods for parsing output files
+  class Parsers
+
+    # @note VirHostMatcher returns true distances that run from 0 to 1, so it doesn't need scaling.
+    # @note VirHostMatcher includes the whole file name as the id of the organism, so we chop off some common endings.
+    def self.vir_host_matcher fname
+      hosts = nil
+
+      host_info = {}
+      File.open(fname, "rt").each_line.with_index do |line, idx|
+        line.chomp!
+        line.sub! /,$/, "" # git rid of trailing commas
+
+        if idx.zero?
+          stat, *hosts = line.split ","
+
+          hosts.map! { |str| BigSimon::Utils.strip_suffix str }
+        else
+          ary   = line.split ","
+          virus = BigSimon::Utils.strip_suffix ary.shift
+
+          # In this case the best value is the lowest distance.
+          dists = ary.map.
+            with_index do |dist, idx|
+            { host: hosts[idx], score: dist.to_f, scaled_score: dist.to_f }
+          end.sort_by { |ht| ht[:scaled_score] }
+
+
+          host_info[virus] = dists
+        end
+      end
+
+      host_info
+    end
+
+    # @note WIsH gives log likelihoods so the scaled value is actually scaled.
+    # @note The viruses and hosts will have the ID rather than the file name.
+    def self.wish fname
+      viruses = nil
+
+      host_info = {}
+
+      hosts = nil
+      File.open(fname, "rt").each_line.map.with_index do |line, idx|
+        line.chomp!
+
+        if idx.zero?
+          ary = line.split("\t")
+          ary.unshift("")
+        else
+          ary = line.split("\t")
+        end
+      end.transpose.each_with_index do |line_ary, idx|
+        if idx.zero?
+          hosts = line_ary.drop(1)
+        else
+          virus = line_ary.shift
+
+          scores      = line_ary.map(&:to_f)
+
+          host_vals = scores.map.with_index do |score, idx|
+            { host: hosts[idx], score: score, scaled_score: 1 - Math.exp(score) }
+          end
+
+          host_info[virus] = host_vals
+        end
+
+        host_info.each do |virus, hosts|
+          hosts.sort_by! { |ht| ht[:scaled_score] }
+        end
+      end
+
+      host_info
+    end
+  end
+
+end

data/lib/big_simon/pipeline.rb

@@ -0,0 +1,64 @@
+module BigSimon
+  class Pipeline
+    # @param collated_results_table { virus => host => score_type => program => score }
+    def self.map_taxa collated_results_table, virus_name_map, host_name_map
+      new_results_table = {}
+
+      collated_results_table.each do |virus_name, host_table|
+        if virus_name_map.include? virus_name
+          new_virus_name = virus_name_map[virus_name]
+        else
+          new_virus_name = virus_name
+        end
+
+        new_results_table[new_virus_name] = {}
+
+        host_table.each do |host_name, score_table|
+          if host_name_map.include? host_name
+            new_host_name = host_name_map[host_name]
+          else
+            new_host_name = host_name
+          end
+
+          new_results_table[new_virus_name][new_host_name] = score_table
+        end
+      end
+
+      new_results_table
+    end
+
+    # @param [Array<Hash>] results_table host info hash tables.  See functions in Parsers class.
+    # @param [Array<String>] programs names of programs generating hash tables (in same order as host_data)
+    def self.collate_host_results results_table, programs
+      Rya::AbortIf.assert results_table.count == programs.count
+
+      virus_host_scores = {}
+      all_viruses       = results_table.reduce(Set.new) { |acc, ht| acc + ht.keys }
+
+      all_viruses.each do |virus|
+        virus_host_scores[virus] = {}
+      end
+
+      results_table.each_with_index do |ht, idx|
+        program = programs[idx]
+
+        ht.each do |virus, host_scores|
+          host_scores.each do |ht|
+            host = ht[:host]
+            score = ht[:score]
+            scaled_score = ht[:scaled_score]
+
+            unless virus_host_scores[virus].has_key? host
+              virus_host_scores[virus][host] = { scores: {}, scaled_scores: {}}
+            end
+
+            virus_host_scores[virus][host][:scores][program] = score
+            virus_host_scores[virus][host][:scaled_scores][program] = scaled_score
+          end
+        end
+      end
+
+      virus_host_scores
+    end
+  end
+end

data/lib/big_simon/runners.rb

@@ -0,0 +1,189 @@
+require "tempfile"
+
+module BigSimon
+  class Runners
+
+    # This one's a bit different as it parses as well and returns original names.
+    # @todo Also do the reverse of each genome in case it's a contig.
+    def self.mummer exe, vir_dir, host_dir, outdir, threads
+      klass = Class.new.extend Rya::CoreExtensions::Math
+      FileUtils.mkdir_p outdir
+
+      # TODO put these all in one file then do it?
+
+      results = {}
+
+      # Takes names in files and puts them to the file names
+      name_map = {}
+
+      Dir.glob(vir_dir + "/*").each do |vir_fname|
+        this_virus_scores = []
+        virus             = nil
+
+        Dir.glob(host_dir + "/*").each do |host_fname|
+          vir_base  = File.basename vir_fname
+          host_base = File.basename host_fname
+          outfname  = File.join outdir, "#{vir_base}___#{host_base}.mummer"
+
+          # -l is min length of a match TODO pull this into a const
+          # -F to force 4 columns
+          cmd = "#{exe} -F " \
+                "-maxmatch " \
+                "-l 15 " \
+                "#{host_fname} " \
+                "#{vir_fname} " \
+                "> #{outfname}"
+
+          Process.run_and_time_it! "Calculating matches", cmd
+
+          # Note there should only be one '>' per file here.
+          host  = nil
+          score = 0
+          File.open(outfname, "rt").each_line.with_index do |line, idx|
+            if idx.zero?
+              this_virus = line.chomp.sub(/^>/, "").sub(/___reverse$/, "").strip
+
+              Rya::AbortIf::abort_unless(this_virus == virus, "OOPS") if virus
+
+              virus ||= this_virus
+            else
+              ary = line.chomp.strip.split(" ")
+              Rya::AbortIf.abort_unless ary.count == 4, "Problem parsing #{outfname} (mummer output)"
+
+              host  = ary[0].sub(/___reverse$/, "").strip
+              len   = ary[3].to_i
+
+              score = len if len > score
+            end
+          end
+
+          this_virus_scores << score
+
+          unless results.has_key? virus
+            results[virus] = []
+          end
+
+          results[virus] << { host: host, score: score, scaled_score: nil }
+
+          FileUtils.rm outfname
+        end
+
+        # This was the original scaling, i.e. per virus
+        # min = 0 # this_virus_scores.min # Technically, this should range from 0 to 15.  Any data missing from this table would give a zero.  TODO we don't actually account for this though.
+        # max = this_virus_scores.max
+        # from = 1
+        # to = 0
+        #
+        # results[virus].each do |host_table|
+        #   host_table[:scaled_score] = klass.scale host_table[:score], min, max, from, to
+        # end
+      end
+
+      all_scores = []
+      results.each do |virus, host_tables|
+        all_scores << host_tables.map { |table| table[:score] }
+      end
+
+      all_scores.flatten!
+      max = all_scores.max
+
+      results.each do |virus, host_tables|
+        host_tables.each do |host_table|
+          host_table[:scaled_score] = klass.scale host_table[:score], 0, max, 1, 0
+        end
+      end
+
+      results
+    end
+
+    def self.vir_host_matcher exe, vir_dir, host_dir, outdir
+      FileUtils.mkdir_p outdir
+
+      cmd = "python #{exe} " \
+      "-v #{vir_dir} " \
+      "-b #{host_dir} " \
+      "-o #{outdir} " \
+      "-d 1" # only compute d2star dissimilarity
+
+      Process.run_and_time_it! "Computing d2star dissimilarity", cmd
+
+      tmp_dir = File.join outdir, "tmp"
+      FileUtils.rm_r tmp_dir if Dir.exist? tmp_dir
+
+      bad_files = %w[d2star_k6_main.html hostTaxa.txt_new.txt]
+      bad_files.each do |fname|
+        path = File.join outdir, fname
+
+        FileUtils.rm path if File.exist? path
+      end
+
+      outf     = File.join outdir, "d2star_k6.csv"
+      new_outf = File.join outdir, "vir_host_matcher.txt"
+      FileUtils.mv outf, new_outf
+
+      new_outf
+    end
+
+    # Runs the WIsH program
+    #
+    # @raise [AbortIf::Exit] if commands fail
+    def self.wish exe, vir_dir, host_dir, outdir, threads
+      model_dir = File.join outdir, "model"
+
+      FileUtils.mkdir_p model_dir
+
+      build_model = "#{exe} " \
+      "-t #{threads} " \
+      "-c build " \
+      "-g #{host_dir} " \
+      "-m #{model_dir}"
+
+      predict = "#{exe} " \
+      "-t #{threads} " \
+      "-c predict " \
+      "-g #{vir_dir} " \
+      "-m #{model_dir} " \
+      "-r #{outdir}"
+
+      Process.run_and_time_it! "Building model", build_model
+      Process.run_and_time_it! "Predicting host", predict
+
+      FileUtils.rm_r model_dir if Dir.exist? model_dir
+
+      outf     = File.join outdir, "llikelihood.matrix"
+      new_outf = File.join outdir, "wish.txt"
+      FileUtils.mv outf, new_outf
+
+      new_outf
+    end
+
+    def self.heatmaps exe, indir, outdir
+      FileUtils.mkdir_p outdir
+
+      fnames = Dir.glob("#{indir}/scores*.txt").map do |in_fname|
+        extname  = File.extname in_fname
+        basename = File.basename in_fname, extname
+
+        out_fname = File.join outdir, "#{basename}.heatmap.pdf"
+
+        [in_fname, out_fname]
+      end
+
+
+      rcode_str = BigSimon::Utils.rcode fnames
+
+      Object::Tempfile.open do |f|
+        f.puts rcode_str
+        f.fsync # ensure no data is buffered
+
+
+        cmd = "#{exe} #{f.path}"
+        Process.run_and_time_it! "Drawing heatmaps", cmd
+      end
+
+      out_fnames = fnames.map(&:last)
+    end
+  end
+end
+
+

data/lib/big_simon/utils.rb

@@ -0,0 +1,108 @@
+module BigSimon
+  # @todo These don't have unit tests yet.
+  # @note Skips any duplicate IDs.  Only keeps the first one.
+  class Utils
+    def self.check_file! fname
+      Rya::AbortIf.abort_if fname && !File.exist?(fname),
+                            "#{fname} doesn't exist!  Try big_simon --help for help."
+    end
+
+    def self.check_opt! opts, arg
+      Rya::AbortIf.abort_unless opts.send(:fetch, "#{arg}_given".to_sym),
+                                "You must specify --#{arg.to_s.tr('_', '-')}.  Try big_simon --help for help."
+    end
+
+    def self.rcode fnames
+      functions = %Q|
+  library(reshape2)
+library(gplots)
+library(RColorBrewer)
+
+file.join <- function(...) {
+    paste(..., sep="/")
+}
+
+draw.heatmap <- function(infname, outfname) {
+    dat <- read.table(infname, header=T, sep="\t")
+
+    wide.dat <- dcast(dat, host ~ virus, value.var="score")
+
+    hosts <- wide.dat[, 1]
+    scores <- wide.dat[, 2:ncol(wide.dat)]
+    scores.numeric <- apply(scores, 2, as.numeric)
+
+    scores.matrix <- as.matrix(scores.numeric)
+
+    rownames(scores.matrix) <- hosts
+
+    palette <- "YlOrBr"
+    col <- colorRampPalette(brewer.pal(n=9, palette))(n = 25)
+    size <- 0.75
+
+    pdf(outfname, height=5, width=8)
+
+    heatmap.2(scores.matrix,
+              trace="none", ## Disable those wonky lines.
+              col=col, ## Set the color.
+
+              ## Size opts
+              margins=c(11, 11), cexRow=size, cexCol=size,
+
+              ## Key labeling
+              key.xlab="Score")
+
+    invisible(dev.off())
+}
+
+|
+
+      drawing = fnames.map do |in_fname, out_fname|
+        %Q{
+
+draw.heatmap("#{in_fname}", "#{out_fname}")
+}
+      end.join
+
+      [functions, drawing].join "\n"
+    end
+
+    def self.scale_log_likelihood ll
+      1 - Math.exp(ll)
+    end
+
+    def self.set_up_tmp_dirs fastas, tmpdir, which
+      Object::FileUtils.mkdir_p tmpdir
+
+      name_map = {}
+      all_ids  = Set.new
+
+      seq_num = -1
+      fastas.each do |fname|
+        ParseFasta::SeqFile.open(fname).each_record do |rec|
+          if all_ids.include? rec.id
+            Rya::AbortIf.logger.warn { "#{rec.id} was seen more than one time!  Duplicate organism IDs are not allowed, so we will only keep the first one." }
+          else
+            all_ids << rec.id
+
+            seq_num += 1
+
+            new_id           = "#{which}_#{seq_num}"
+            name_map[new_id] = rec.id
+
+            outfname = File.join tmpdir, "#{new_id}.fa"
+
+            File.open(outfname, "w") do |f|
+              f.puts rec
+            end
+          end
+        end
+      end
+
+      [name_map, all_ids]
+    end
+
+    def self.strip_suffix fname
+      fname.sub /.fasta$|.fa$/, ""
+    end
+  end
+end

data/lib/big_simon/version.rb

@@ -1,3 +1,15 @@
 module BigSimon
-  VERSION = "0.0.1"
+  VERSION = "0.1.0"
+
+  COPYRIGHT = "2018 Ryan Moore"
+  CONTACT   = "moorer@udel.edu"
+  WEBSITE   = "https://github.com/mooreryan/InteinFinder"
+  LICENSE   = "GPLv3"
+
+  VERSION_BANNER =
+      "  # Version:   v#{VERSION}
+  # Copyright: #{COPYRIGHT}
+  # Contact:   #{CONTACT}
+  # License:   #{LICENSE}"
+
 end

data/lib/big_simon.rb

@@ -1,100 +1,27 @@
 require "rya"
+require "set"
+require "pp"
 
 require "big_simon/version"
 
+require "big_simon/utils"
+
+require "big_simon/runners"
+require "big_simon/parsers"
+require "big_simon/pipeline"
+
 Time.extend Rya::CoreExtensions::Time
 Process.extend Rya::CoreExtensions::Process
+Array.include Rya::CoreExtensions::Array
+Math.extend Rya::CoreExtensions::Math
 
 module BigSimon
-
-  # Project directories
   ROOT       = File.join __dir__, ".."
   BIN        = File.join ROOT, "vendor", "bin", "mac"
   SPEC       = File.join ROOT, "spec"
   TEST_FILES = File.join SPEC, "test_files"
-
-  class Parsers
-
-    def self.vir_host_matcher fname
-      hosts = nil
-
-      host_info = {}
-      File.open(fname, "rt").each_line.with_index do |line, idx|
-        line.chomp!
-        line.sub! /,$/, "" # git rid of trailing commas
-
-        if idx.zero?
-          stat, *hosts = line.split ","
-        else
-          ary   = line.split ","
-          virus = ary.shift
-
-          dists = ary.map.
-              with_index { |dist, idx| [hosts[idx], dist.to_f] }.
-              sort_by { |_, dist| dist }
-
-          best_host = dists[0][0]
-
-          host_info[virus] = {
-              best: best_host,
-              all:  dists
-          }
-        end
-      end
-
-      host_info
-    end
-  end
-
-  class Runners
-
-    # Runs the WIsH program
-    #
-    # @raise [AbortIf::Exit] if commands fail
-    def self.wish exe, vir_dir, host_dir, outdir, threads
-      model_dir = File.join outdir, "model"
-
-      FileUtils.mkdir_p model_dir
-
-      build_model = "#{exe} " \
-      "-t #{threads} " \
-      "-c build " \
-      "-g #{host_dir} " \
-      "-m #{model_dir}"
-
-      predict = "#{exe} " \
-      "-t #{threads} " \
-      "-c predict " \
-      "-g #{vir_dir} " \
-      "-m #{model_dir} " \
-      "-r #{outdir} -b"
-
-      Process.run_and_time_it! "Building model", build_model
-      Process.run_and_time_it! "Predicting host", predict
-
-      FileUtils.rm_r model_dir if Dir.exist? model_dir
-    end
-
-    def self.vir_host_matcher exe, vir_dir, host_dir, outdir
-      FileUtils.mkdir_p outdir
-
-      cmd = "python #{exe} " \
-      "-v #{vir_dir} " \
-      "-b #{host_dir} " \
-      "-o #{outdir} " \
-      "-d 1" # only compute d2star dissimilarity
-
-      Process.run_and_time_it! "Computing d2star dissimilarity", cmd
-
-      tmp_dir = File.join outdir, "tmp"
-      FileUtils.rm_r tmp_dir if Dir.exist? tmp_dir
-
-      bad_files = %w[d2star_k6_main.html hostTaxa.txt_new.txt]
-      bad_files.each do |fname|
-        path = File.join outdir, fname
-
-        FileUtils.rm path if File.exist? path
-      end
-    end
-  end
+  WISH       = File.join BIN, "WIsH"
+  VHM        = File.join BIN, "vhm.py"
+  MUMMER     = File.join BIN, "mummer"
+  RSCRIPT = "Rscript"
 end

data/vendor/bin/mac/vhm.py

@@ -46,8 +46,9 @@ if not os.path.exists(tmpDir) :
 filelog = open(os.path.join(tmpDir, 'vhm.log'), 'w')
 
 ## name length ##
-nameLen = 93 - len(options.outDir)
+# nameLen = 93 - len(options.outDir)
 #### possibly because of the kmercount folder name for each contig is too long?
+nameLen = 99999999999999
 
 
 #################### 0: preparation ############################
@@ -211,7 +212,7 @@ if options.hostTaxaFile is None :
 	hostTaxaFileWrite = open(hostTaxaFile, 'w') ## make file blank
 	hostTaxaFileWrite.close()
 	hostTaxaFileWrite = open(hostTaxaFile, 'a')
-	
+
 	hostTaxaFileWrite.write("hostNCBIName	hostName	hostSuperkingdom	hostPhylum	hostClass	hostOrder	hostFamily	hostGenus	hostSpecies\n")
 	for currentFileName in hostFaList :
 		if currentFileName.startswith('.') :
@@ -235,7 +236,7 @@ else :
 	hostTaxaTable = numpy.genfromtxt(options.hostTaxaFile,delimiter="\t", dtype=str)
 	hostTaxaTable[hostTaxaTable=='']='unknown'
 	numpy.savetxt(hostTaxaFile, hostTaxaTable, fmt="%s", delimiter='\t', newline='\n')
-    
+
 filelog.flush()
 
 #################### 1: count kmer and prepare list files ############################
@@ -259,7 +260,7 @@ for currentFileName in virusFaList :
 	filelog.write("Step 1: counting kmers for virus " + currentFileNameS + "\n")
 	for w in range(1, (kmax+1)) :
 		currentFilePath = os.path.join(options.virusFaDir, currentFileName)
-    
+
 		currentKmerCountPath = os.path.join(kmerCountPath, currentFileNameS)
 		cmdKmer = countKmerOut + " -l -k " + str(w) + \
 										" -i " + currentFilePath +\
@@ -278,9 +279,9 @@ for currentFileName in virusFaList :
 		sys.stderr.write( "ERROR in counting kmers for " + currentFileNameS + "\n")
 		filelog.write( "ERROR in counting kmers for " + currentFileNameS + "\n")
 		sys.exit(0)
-  
+
 	filelog.flush()
-  
+
 	end_time = time.time()
 	count += 1
 	#sys.stdout.write(str(end_time - start_time) + "s for " + str(count) + " seqs \n")
@@ -329,9 +330,9 @@ for currentFileName in hostFaList :
 		sys.stderr.write( "ERROR in counting kmers for " + currentFileNameS + "\n")
 		filelog.write( "ERROR in counting kmers for " + currentFileNameS + "\n")
 		sys.exit(0)
-  
+
 	filelog.flush()
-  
+
 	end_time = time.time()
 	count += 1
 	#sys.stdout.write(str(end_time - start_time) + "s for " + str(count) + " seqs \n")
@@ -366,7 +367,7 @@ for c in iter(cmdCptMeasureOut.stderr.readline, b''):
 	sys.stdout.write(c.decode("utf-8"))
 	filelog.write(c.decode("utf-8"))
 	filelog.flush()
-  
+
 	end_time = time.time()
 	count += 1
 	sys.stdout.write("   (Average time for computing dissimilarities for one virus-host pair: " + str(round((end_time - start_time)/count/len(virusFaList), 4)) + "s) \n")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: big_simon
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Ryan Moore
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-28 00:00:00.000000000 Z
+date: 2018-07-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,24 +52,66 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: parse_fasta
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.5.2
 - !ruby/object:Gem::Dependency
   name: rya
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.3
+        version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.3
+        version: 0.4.0
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.3
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.1.3
 description: Viral host discovery pipeline.
 email:
 - moorer@udel.edu
-executables: []
+executables:
+- big_simon
+- ranks
 extensions: []
 extra_rdoc_files: []
 files:
@@ -80,17 +122,25 @@ files:
 - COPYING
 - Gemfile
 - Gemfile.lock
+- Makefile
 - README.md
 - Rakefile
 - big_simon.gemspec
 - bin/console
 - bin/setup
+- exe/big_simon
+- exe/ranks
 - lib/big_simon.rb
+- lib/big_simon/parsers.rb
+- lib/big_simon/pipeline.rb
+- lib/big_simon/runners.rb
+- lib/big_simon/utils.rb
 - lib/big_simon/version.rb
 - vendor/bin/mac/WIsH
 - vendor/bin/mac/computeMeasure.out
 - vendor/bin/mac/computeMeasure_onlyd2star.out
 - vendor/bin/mac/countKmer.out
+- vendor/bin/mac/mummer
 - vendor/bin/mac/vhm.py
 homepage: https://github.com/mooreryan/big_simon
 licenses: []