big_simon 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9c81d1057b304f170cdf1dd6a7551bd981efc232
-  data.tar.gz: c0550efe308bc0080c9f14ce65057b100ff2f32b
+  metadata.gz: 95ca94aff107e2545dd26ab671c2ed125cb89164
+  data.tar.gz: 8cb9098b4462d6702ecd692e7261abb9c1c7c26a
 SHA512:
-  metadata.gz: 652abc1ab4507ed51012cde1a541a8e5ce75e30a2c2d8b9ef4cb0363e49bb8ea4f088dcb4786bb5f9f466f8f0ed0d2f2e6d9b446aa3a4c8fd2a6a8f4370f6a78
-  data.tar.gz: 3a82d66018a1dea01afb9f0230a9f8ad6ac90d96d5862a79b57ecb03f954eedf9183c2dc9d03b6f98a6a0675b3293577c4e5001fa7f792bbe68ebefe397d47f8
+  metadata.gz: be890e07e1ca248da8e5b80637b25d00c20185d4710229cc9c72d564abc930d4eb734ad53f42842a479827e64205ee1f53f5691b6527d03b9cc41b9b8656ae85
+  data.tar.gz: ac30f597de91303e3b8040ecfde48f98d56c343aadfa3a1085d77fbb55692fb055569734470812144d716e73ee8c94265c732a07462bd8490c2b937572961062

data/.gitignore

@@ -30,4 +30,12 @@ spec/test_files/outdir_for_heatmaps/outdir/
 
 stats.tmp.r
 
-scratch
+scratch
+
+
+
+spec/test_files/homology_files/output/*homology*
+
+pec/test_files/mummer_files/output/mummer_out.txt
+
+tmp.prodigal.stdin.5129

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    big_simon (0.1.1)
+    big_simon (0.2.0)
+      parallel
       parse_fasta (~> 2.5, >= 2.5.2)
       rya (~> 0.4.0)
       trollop (~> 2.1, >= 2.1.3)
@@ -11,6 +12,7 @@ GEM
   specs:
     abort_if (0.2.0)
     diff-lcs (1.3)
+    parallel (1.12.1)
     parse_fasta (2.5.2)
     rake (10.5.0)
     rspec (3.7.0)

data/big_simon.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
 
+  spec.add_runtime_dependency "parallel"
   spec.add_runtime_dependency "parse_fasta", "~> 2.5", ">= 2.5.2"
   spec.add_runtime_dependency "rya", "~> 0.4.0"
   spec.add_runtime_dependency "trollop", "~> 2.1", ">= 2.1.3"

data/exe/big_simon

@@ -54,7 +54,8 @@ Rya::AbortIf.abort_unless opts[:threads] > 0,
 programs = [
   "WIsH",
   "VirHostMatcher",
-  "mummer"
+  "mummer",
+  "homology",
 ]
 
 outdir       = opts[:outdir]
@@ -71,87 +72,6 @@ tmpdir_host  = File.join tmpdir, "host"
 # all_predictions_fname = File.join outdir, "scores_all.txt"
 mean_scaled_scores_fname = File.join outdir, "scores_scaled.mean.txt"
 
-# virus_recs, host_recs = [], []
-
-# Tempfile.open do |vir_f|
-#   Tempfile.open do |host_f|
-#     virus_fnames.each do |fname|
-#       ParseFasta::SeqFile.open(fname).each_record do |rec|
-#         vir_f.puts rec
-#
-#         vir_f.puts ">#{rec.id}___reverse\n#{rec.seq.reverse}"
-#       end
-#     end
-#
-#     host_fnames.each do |fname|
-#       ParseFasta::SeqFile.open(fname).each_record do |rec|
-#         host_f.puts rec
-#         host_f.puts ">#{rec.id}___reverse\n#{rec.seq.reverse}"
-#       end
-#     end
-#
-#     vir_f.fsync
-#     host_f.fsync
-#
-#     cmd = "mummer -maxmatch -l 15 #{host_f.path} #{vir_f.path} > /Users/moorer/Desktop/mummer.OUT"
-#     Process.run_and_time_it! "MUMMER", cmd
-#   end
-# end
-#
-# header   = nil
-# hits     = []
-# hit_info = {}
-# virus = nil
-#
-# File.open("/Users/moorer/Desktop/mummer.OUT", "rt").each_line.with_index do |line, idx|
-#   if line.start_with? '>'
-#     virus = line.chomp.sub(/^>/, "").sub(/___reverse$/, "").strip
-#
-#     unless hit_info.has_key? virus
-#       hit_info[virus] = {}
-#     end
-#   else
-#     host, _, _, len = line.chomp.strip.split(" ")
-#     host = host.sub(/___reverse$/, "").strip
-#
-#     unless hit_info[virus].has_key? host
-#       hit_info[virus][host] = -1
-#     end
-#
-#     hit_info[virus][host] = len.to_i if len.to_i > hit_info[virus][host]
-#   end
-# end
-#
-# puts
-#
-# hh = hit_info.map do |virus, info|
-#   [virus, info.to_a.sort_by {|gen, len| len}.reverse]
-# end
-#
-# pp hh
-
-# hh = hit_info.map do |virus, info|
-#   [virus, info.to_a.sort_by { |host, hit_len| hit_len }.reverse
-#
-# end
-# p hit_info
-
-scores_files = {}
-programs.each do |program|
-  raw_fname    = File.join outdir, "scores_raw.#{program}.txt"
-  scaled_fname = File.join outdir, "scores_scaled.#{program}.txt"
-
-  scores_files[program] = {
-    raw:    File.open(raw_fname, "w"),
-    scaled: File.open(scaled_fname, "w")
-  }
-end
-
-scores_files.each do |program, files|
-  files.each do |name, file|
-    file.puts %w[virus host score].join "\t"
-  end
-end
 
 name_map_virus, all_ids_virus = BigSimon::Utils.set_up_tmp_dirs virus_fnames, tmpdir_virus, "virus"
 name_map_host, all_ids_host   = BigSimon::Utils.set_up_tmp_dirs host_fnames, tmpdir_host, "host"
@@ -162,12 +82,38 @@ vhm_outf  = BigSimon::Runners.vir_host_matcher BigSimon::VHM, tmpdir_virus, tmpd
 # TODO separate the parser from the runner for mummer.
 host_info_mummer = BigSimon::Runners.mummer BigSimon::MUMMER, tmpdir_virus, tmpdir_host, tmpdir, threads
 
+puts "mummer"
+pp host_info_mummer
+puts
+
+# TODO separate the parser from the runner for homology
+host_info_homology = BigSimon::Runners.homology tmpdir_virus, tmpdir_host, tmpdir, threads
+
+puts "homology"
+pp host_info_homology
+puts
+
+
+
 host_info_wish = BigSimon::Parsers.wish wish_outf
 host_info_vhm  = BigSimon::Parsers.vir_host_matcher vhm_outf
 
-host_info_simple_names = BigSimon::Pipeline.collate_host_results [host_info_wish, host_info_vhm, host_info_mummer], programs
+puts "wish"
+pp host_info_wish
+puts
+
+puts "vhm"
+pp host_info_vhm
+puts
+
+
+host_info_simple_names = BigSimon::Pipeline.collate_host_results [host_info_wish, host_info_vhm, host_info_mummer, host_info_homology], programs
 host_info              = BigSimon::Pipeline.map_taxa host_info_simple_names, name_map_virus, name_map_host
 
+puts
+pp host_info_simple_names
+puts
+
 puts
 pp host_info
 puts
@@ -176,6 +122,25 @@ puts
 # File.open all_predictions_fname, "w" do |f|
 #   f.puts %w[virus host program score scaled.score].join "\t"
 
+scores_files = {}
+programs.each do |program|
+  raw_fname    = File.join outdir, "scores_raw.#{program}.txt"
+  scaled_fname = File.join outdir, "scores_scaled.#{program}.txt"
+
+  scores_files[program] = {
+    raw:    File.open(raw_fname, "w"),
+    scaled: File.open(scaled_fname, "w")
+  }
+end
+
+scores_files.each do |program, files|
+  files.each do |name, file|
+    puts "LALA 3: #{[program, name, file]} #{%w[virus host score].join}"
+
+    file.puts %w[virus host score].join "\t"
+  end
+end
+
 host_info.each do |virus, h1|
   h1.each do |host, h2|
     lines = {}
@@ -183,6 +148,8 @@ host_info.each do |virus, h1|
     h2[:scores].each do |program, score|
       lines[[virus, host, program]] = [score]
 
+      puts "LALA: #{[virus, host, score]}"
+
       scores_files[program][:raw].puts [virus, host, score].join "\t"
     end
 
@@ -190,6 +157,9 @@ host_info.each do |virus, h1|
     h2[:scaled_scores].each do |program, score|
       lines[[virus, host, program]] << score
 
+      puts "LALA 2: #{[virus, host, score]}"
+
+
       scores_files[program][:scaled].puts [virus, host, score].join "\t"
     end
 

data/lib/big_simon/runners.rb

@@ -1,5 +1,6 @@
 require "tempfile"
 require "parse_fasta"
+require "parallel"
 
 module BigSimon
   class Runners
@@ -52,12 +53,15 @@ module BigSimon
           vir_f.fsync
           host_f.fsync
 
-          cmd = "mummer -threads #{threads} -qthreads #{threads} -maxmatch -l 15 #{host_f.path} #{vir_f.path} > #{mummer_outfname}"
+          # -k 3 index every third position in reference (broken now, bug in mummer)
+          # -n -k 3 -threads 3
+          # -n match only A C T G
+          cmd = "mummer -n -threads #{threads} -qthreads #{threads} -maxmatch -l 15 #{host_f.path} #{vir_f.path} > #{mummer_outfname}"
           Process.run_and_time_it! "MUMMER", cmd
         end
       end
 
-      virus = nil
+      virus             = nil
       overall_max_score = 0
       File.open(mummer_outfname, "rt").each_line.with_index do |line, idx|
         line.chomp!
@@ -75,7 +79,7 @@ module BigSimon
           else
             ary = line.strip.split " "
 
-            host = ary[0].sub(/___reverse$/, "").strip
+            host  = ary[0].sub(/___reverse$/, "").strip
             score = ary[3].to_i
 
             Rya::AbortIf.assert hit_table[virus].has_key?(host)
@@ -88,7 +92,7 @@ module BigSimon
             hit_table[virus][host] = score if score > hit_table[virus][host]
 
             # Track the overall max for scaling.
-            overall_max_score = score if score > overall_max_score
+            overall_max_score      = score if score > overall_max_score
           end
         end
       end
@@ -203,6 +207,121 @@ module BigSimon
       results
     end
 
+    # For scoring homology-ness, I just sum the bitscore for all significant hits for all genomes.
+    #
+    # @note I will make the specified outdir if it doesn't exist.
+    # @note Assumes that the files end with *.fa
+    # @note Assumes that the file names match the IDs.  This SHOULD be taken care of by the big_simon program.
+    # @todo assert that fname thing matches sequence ID name.
+    def self.homology vir_dir, host_dir, outdir, threads
+      FileUtils.mkdir_p outdir
+
+      host_orfs          = File.join outdir, "host_orfs.homology"
+      host_orfs_blast_db = host_orfs + ".blast_db.homology"
+
+      # Call ORFs on Hosts
+      cmd = "cat #{host_dir}/*.fa | #{BigSimon::PRODIGAL} " \
+      "-d #{host_orfs} " \
+      "> /dev/null"
+
+      Process.run_and_time_it! "Predicting host ORFs", cmd
+
+      # Make blast db's for the host genes.
+      cmd = "#{BigSimon::MAKEBLASTDB} " \
+      "-in #{host_orfs} " \
+      "-out #{host_orfs_blast_db} " \
+      "-dbtype nucl"
+
+      Process.run_and_time_it! "Making host blast db", cmd
+
+      vir_genome_fnames = Dir.glob(vir_dir + "/*.fa")
+
+      blast_info = Parallel.map(vir_genome_fnames, in_processes: threads) do |vir_genome_fname|
+        vir_orfs      = File.join outdir, File.basename(vir_genome_fname) + ".vir_orfs.homology"
+        blast_results = File.join outdir, File.basename(vir_genome_fname) + ".blast_results.homology"
+
+        # this will be used as a viral ID.
+        vir_simple_fname              = File.basename vir_genome_fname, ".fa"
+        blast_table                   = {}
+        blast_table[vir_simple_fname] = Hash.new 0
+
+        # Call ORFs on the virus.
+        cmd = "#{BigSimon::PRODIGAL} " \
+        "-d #{vir_orfs} -p meta -i #{vir_genome_fname} " \
+        "> /dev/null"
+
+        Process.run_and_time_it! "Predicting ORFs for #{File.basename vir_genome_fname}", cmd
+
+        # Blast the ORFs against genomes.
+        cmd = "#{BigSimon::BLASTN} -query #{vir_orfs} -db #{host_orfs_blast_db} -outfmt 6 -evalue 0.01 -word_size 11 -out #{blast_results}"
+        Process.run_and_time_it! "Blasting ORFs for #{File.basename vir_genome_fname}", cmd
+
+        # Remove ORFs file
+        FileUtils.rm vir_orfs if File.exist? vir_orfs
+
+        Rya::AbortIf.logger.info { "Parsing #{blast_results}" }
+        # Parse the blast.
+
+        File.open(blast_results, "rt").each_line do |line|
+          ary = line.chomp.split "\t"
+
+          # The .sub() is to remove the annotation that prodigal gives.
+          vir_id  = ary[0].sub(/_[0-9]+$/, "")
+          host_id = ary[1].sub(/_[0-9]+$/, "")
+          score   = ary[11].to_f
+
+          Rya::AbortIf.assert blast_table.has_key?(vir_id), "blast_table: got #{vir_id} should have been #{vir_simple_fname}"
+
+          blast_table[vir_id][host_id] += score
+        end
+
+        # Remove blast file
+        # FileUtils.rm_r blast_results if File.exist? blast_results
+
+        # Again, we're assuming the input is .fa, which the big_simon program SHOULD ensure.  TODO check these things with assertions.
+        simple_vir_name = File.basename vir_genome_fname.sub(/.fa$/, "")
+
+        [simple_vir_name, blast_table]
+      end
+
+      collated_blast_table = {}
+      host_simple_names    = Dir.glob(host_dir + "/*.fa").map { |fname| File.basename(fname, ".fa") }
+
+      Rya::AbortIf.assert host_simple_names.length == host_simple_names.uniq.length, "host simple names are not unique"
+
+      Rya::AbortIf.logger.info { "Collating blast results" }
+
+      # Get max score
+      max_score = -1
+      blast_info.each do |_, blast_table|
+        blast_table.each do |vir_id, host_scores|
+          this_max  = host_scores.values.max || -1 # sometimes there are no hits at all
+
+          max_score = this_max if this_max > max_score
+        end
+      end
+      Rya::AbortIf.assert max_score > -1, "didn't get any scores"
+
+
+      klass = Class.new.extend Rya::CoreExtensions::Math
+      blast_info.each do |simple_vir_name, blast_table|
+        blast_table.each do |vir_id, host_scores|
+          collated_blast_table[vir_id] = []
+
+          host_simple_names.each do |host_id|
+            scaled_score = klass.scale host_scores[host_id], 0, max_score, 1, 0
+
+            host_table = { host: host_id, score: host_scores[host_id], scaled_score: scaled_score }
+            collated_blast_table[vir_id] << host_table
+          end
+        end
+      end
+
+      pp collated_blast_table
+
+      collated_blast_table
+    end
+
     def self.vir_host_matcher exe, vir_dir, host_dir, outdir
       FileUtils.mkdir_p outdir
 
@@ -273,13 +392,13 @@ module BigSimon
 
         out_fname = File.join outdir, "#{basename}.heatmap.pdf"
 
-        [in_fname, out_fname]
+        [File.absolute_path(in_fname), File.absolute_path(out_fname)]
       end
 
 
       rcode_str = BigSimon::Utils.rcode fnames
 
-      Object::Tempfile.open do |f|
+      Object::File.open(File.join(outdir, "RCODE.r"), "w") do |f|
         f.puts rcode_str
         f.fsync # ensure no data is buffered
 

data/lib/big_simon/utils.rb

@@ -70,6 +70,7 @@ draw.heatmap("#{in_fname}", "#{out_fname}")
       1 - Math.exp(ll)
     end
 
+    # @note I also rename all the sequences in the tmp fasta files with the new ID.
     def self.set_up_tmp_dirs fastas, tmpdir, which
       Object::FileUtils.mkdir_p tmpdir
 
@@ -92,7 +93,7 @@ draw.heatmap("#{in_fname}", "#{out_fname}")
             outfname = File.join tmpdir, "#{new_id}.fa"
 
             File.open(outfname, "w") do |f|
-              f.puts rec
+              f.puts ">#{new_id}\n#{rec.seq}" # TODO HERE
             end
           end
         end

data/lib/big_simon/version.rb

@@ -1,5 +1,5 @@
 module BigSimon
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 
   COPYRIGHT = "2018 Ryan Moore"
   CONTACT   = "moorer@udel.edu"

data/lib/big_simon.rb

@@ -20,8 +20,14 @@ module BigSimon
   BIN        = File.join ROOT, "vendor", "bin", "mac"
   SPEC       = File.join ROOT, "spec"
   TEST_FILES = File.join SPEC, "test_files"
+
+  # Programs
   WISH       = File.join BIN, "WIsH"
   VHM        = File.join BIN, "vhm.py"
   MUMMER     = File.join BIN, "mummer"
   RSCRIPT = "Rscript"
+
+  BLASTN = File.join BIN, "blastn"
+  MAKEBLASTDB = File.join BIN, "makeblastdb"
+  PRODIGAL = File.join BIN, "prodigal"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: big_simon
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Ryan Moore
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-07-30 00:00:00.000000000 Z
+date: 2018-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: parse_fasta
   requirement: !ruby/object:Gem::Requirement
@@ -137,10 +151,13 @@ files:
 - lib/big_simon/utils.rb
 - lib/big_simon/version.rb
 - vendor/bin/mac/WIsH
+- vendor/bin/mac/blastn
 - vendor/bin/mac/computeMeasure.out
 - vendor/bin/mac/computeMeasure_onlyd2star.out
 - vendor/bin/mac/countKmer.out
+- vendor/bin/mac/makeblastdb
 - vendor/bin/mac/mummer
+- vendor/bin/mac/prodigal
 - vendor/bin/mac/vhm.py
 homepage: https://github.com/mooreryan/big_simon
 licenses: []