miga-base 0.7.4.0 → 0.7.5.0

data/utils/distance/pipeline.rb

@@ -1,13 +1,12 @@
-
 # High-end pipelines for DistanceRunner
 module MiGA::DistanceRunner::Pipeline
-
   # Recursively classify the dataset, returning an Array with two entries:
   # classification and cluster number
   def classify(clades, classif, metric, result_fh, val_cls = nil)
     dir = File.expand_path(classif, clades)
     med = File.expand_path('miga-project.medoids', dir)
-    return [classif,val_cls] unless File.size? med
+    return [classif, val_cls] unless File.size? med
+
     max_val = 0
     val_med = ''
     val_cls = nil
@@ -35,6 +34,7 @@ module MiGA::DistanceRunner::Pipeline
     $stderr.puts "Building medoids tree (metric = #{metric})"
     db = query_db(metric)
     return unless File.size? db
+
     out_base = File.expand_path(dataset.name, home)
     ds_matrix = "#{out_base}.txt"
     ds_matrix_fh = File.open(ds_matrix, 'w')
@@ -43,7 +43,7 @@ module MiGA::DistanceRunner::Pipeline
     seq2 = []
     foreach_in_db(db, metric) do |r|
       seq2 << r[0]
-      ds_matrix_fh.puts r[0,3].join("\t")
+      ds_matrix_fh.puts r[0, 3].join("\t")
     end
     # Find all values among visited datasets in ref_project
     ref_r = ref_project.result("#{metric}_distances") or return
@@ -51,7 +51,8 @@ module MiGA::DistanceRunner::Pipeline
       fh.each_line do |ln|
         r = ln.chomp.split("\t")
         next unless seq2.include?(r[1]) or seq2.include?(r[2])
-        ds_matrix_fh.puts r[1,3].join("\t")
+
+        ds_matrix_fh.puts r[1, 3].join("\t")
       end
     end
     ds_matrix_fh.close
@@ -74,11 +75,12 @@ module MiGA::DistanceRunner::Pipeline
     dataset.add_result(from_ref_project ? :taxonomy : :distances, true)
     cr = dataset.closest_relatives(1, from_ref_project)
     return if cr.nil? or cr.empty?
+
     tax = ref_project.dataset(cr[0][0]).metadata[:tax] || {}
 
     # Run the test for each rank
     tax_test = MiGA::TaxDist.aai_pvalues(cr[0][1], :intax, engine: opts[:aai_p])
-    r = tax_test.map do |k,v|
+    r = tax_test.map do |k, v|
       sig = ''
       [0.5, 0.1, 0.05, 0.01].each { |i| sig << '*' if v < i }
       [MiGA::Taxonomy.LONG_RANKS[k], (tax[k] || '?'), v, sig]
@@ -99,10 +101,11 @@ module MiGA::DistanceRunner::Pipeline
   def transfer_taxonomy(tax)
     $stderr.puts "Transferring taxonomy"
     return if tax.nil?
+
     pval = (project.metadata[:tax_pvalue] || 0.05).to_f
-    tax_a = tax.
-      select { |i| i[1] != '?' && i[2] <= pval }.
-      map { |i| i[0,2].join(':') }
+    tax_a = tax
+            .select { |i| i[1] != '?' && i[2] <= pval }
+            .map { |i| i[0, 2].join(':') }
     dataset.metadata[:tax] = MiGA::Taxonomy.new(tax_a)
     dataset.save
   end

data/utils/distance/runner.rb

@@ -1,13 +1,10 @@
-
 require_relative 'base.rb'
 require_relative 'temporal.rb'
 require_relative 'database.rb'
 require_relative 'commands.rb'
 require_relative 'pipeline.rb'
 
-
 class MiGA::DistanceRunner
-
   include MiGA::DistanceRunner::Temporal
   include MiGA::DistanceRunner::Database
   include MiGA::DistanceRunner::Commands
@@ -16,7 +13,7 @@ class MiGA::DistanceRunner
   attr_reader :project, :ref_project, :dataset, :opts, :home
   attr_reader :tmp, :tmp_dbs, :dbs, :db_counts
 
-  def initialize(project_path, dataset_name, opts_hash={})
+  def initialize(project_path, dataset_name, opts_hash = {})
     @opts = opts_hash
     @project = MiGA::Project.load(project_path) or
       raise "No project at #{project_path}"
@@ -30,7 +27,7 @@ class MiGA::DistanceRunner
     @opts[:aai_save_rbm] ||= ENV.fetch('MIGA_AAI_SAVE_RBM') do
       project.is_clade? ? 'save-rbm' : 'no-save-rbm'
     end
-    @opts[:thr] ||= ENV.fetch('CORES'){ 2 }.to_i
+    @opts[:thr] ||= ENV.fetch('CORES') { 2 }.to_i
     if opts[:run_taxonomy] and project.metadata[:ref_project]
       ref_path = project.metadata[:ref_project]
       @home = File.expand_path('05.taxonomy', @home)
@@ -60,6 +57,7 @@ class MiGA::DistanceRunner
   def go!
     $stderr.puts "Launching analysis"
     return if dataset.is_multi?
+
     Dir.mktmpdir do |tmp_dir|
       @tmp = tmp_dir
       create_temporals
@@ -76,12 +74,13 @@ class MiGA::DistanceRunner
     # first-come-first-serve traverse
     ref_project.each_dataset do |ds|
       next if !ds.is_ref? or ds.is_multi? or ds.result(:essential_genes).nil?
+
       puts "[ #{Time.now} ] #{ds.name}"
       ani_after_aai(ds)
     end
 
     # Finalize
-    [:haai, :aai, :ani].each{ |m| checkpoint! m if db_counts[m] > 0 }
+    [:haai, :aai, :ani].each { |m| checkpoint! m if db_counts[m] > 0 }
   end
 
   ##
@@ -104,14 +103,15 @@ class MiGA::DistanceRunner
     # Calculate all the AAIs/ANIs against the lowest subclade (if classified)
     par_dir = File.dirname(File.expand_path(classif, res.dir))
     par = File.expand_path('miga-project.classif', par_dir)
-    closest = {dataset: nil, ani: 0.0}
+    closest = { dataset: nil, ani: 0.0 }
     if File.size? par
       File.open(par, 'r') do |fh|
         fh.each_line do |ln|
           r = ln.chomp.split("\t")
           next unless r[1].to_i == val_cls
+
           ani = ani_after_aai(ref_project.dataset(r[0]), 80.0)
-          closest = {ds: r[0], ani: ani} unless ani.nil? or ani < closest[:ani]
+          closest = { ds: r[0], ani: ani } unless ani.nil? or ani < closest[:ani]
         end
       end
     end
@@ -119,14 +119,14 @@ class MiGA::DistanceRunner
     # Calculate all the AAIs/ANIs against the closest ANI95-clade (if AAI > 80%)
     cl_path = res.file_path :clades_ani95
     if !cl_path.nil? and File.size? cl_path and tsk[0] == :clade_finding
-      File.foreach(cl_path).
-        map  { |i| i.chomp.split(',') }.
-        find( lambda{[]} ){ |i| i.include? closest[:ds] }.
-        each { |i| ani_after_aai(ref_project.dataset(i), 80.0) }
+      File.foreach(cl_path)
+          .map { |i| i.chomp.split(',') }
+          .find(lambda { [] }) { |i| i.include? closest[:ds] }
+          .each { |i| ani_after_aai(ref_project.dataset(i), 80.0) }
     end
 
     # Finalize
-    [:haai, :aai, :ani].each{ |m| checkpoint! m if db_counts[m] > 0 }
+    [:haai, :aai, :ani].each { |m| checkpoint! m if db_counts[m] > 0 }
     build_medoids_tree(tsk[1])
     transfer_taxonomy(tax_test)
   end
@@ -135,6 +135,7 @@ class MiGA::DistanceRunner
   def go_taxonomy!
     $stderr.puts "Launching taxonomy analysis"
     return unless project.metadata[:ref_project]
+
     go_query! # <- yeah, it's actually the same, just different ref_project
   end
 end

data/utils/distance/temporal.rb

@@ -1,9 +1,7 @@
-
 require 'tmpdir'
 require 'zlib'
 
 module MiGA::DistanceRunner::Temporal
-
   # Copy input files to the (local) temporal folder
   def create_temporals
     rf = {
@@ -43,7 +41,7 @@ module MiGA::DistanceRunner::Temporal
   def checkpoint!(metric)
     $stderr.puts "Checkpoint (metric = #{metric})"
     SQLite3::Database.new(tmp_dbs[metric]) do |conn|
-      conn.execute("select count(*) from #{metric==:haai ? :aai : metric}")
+      conn.execute("select count(*) from #{metric == :haai ? :aai : metric}")
     end
     FileUtils.cp(tmp_dbs[metric], dbs[metric])
     @db_counts[metric] = 0

data/utils/distances.rb

@@ -4,6 +4,6 @@ require_relative 'distance/runner.rb'
 
 dataset = ARGV.shift
 project = ARGV.shift
-opts = Hash[ ARGV.map{ |i| i.split("=",2).tap{ |j| j[0] = j[0].to_sym } } ]
+opts = Hash[ARGV.map { |i| i.split("=", 2).tap { |j| j[0] = j[0].to_sym } }]
 runner = MiGA::DistanceRunner.new(dataset, project, opts)
 runner.go!

data/utils/domain-ess-genes.rb

@@ -6,10 +6,10 @@ domain = ARGV.shift
 
 def quality(hsh)
   q = {}
-  q[:found] = hsh.values.map{ |i| i==0 ? 0 : 1 }.inject(:+)
-  q[:multi] = hsh.values.map{ |i| i==0 ? 0 : i-1 }.inject(:+)
-  q[:cmp] = 100.0*q[:found].to_f/hsh.size
-  q[:cnt] = 100.0*q[:multi].to_f/hsh.size
+  q[:found] = hsh.values.map { |i| i == 0 ? 0 : 1 }.inject(:+)
+  q[:multi] = hsh.values.map { |i| i == 0 ? 0 : i - 1 }.inject(:+)
+  q[:cmp] = 100.0 * q[:found].to_f / hsh.size
+  q[:cnt] = 100.0 * q[:multi].to_f / hsh.size
   q
 end
 
@@ -39,7 +39,7 @@ end
 # Find expected genes for domain
 n_dom = Hash[
   `HMM.essential.rb -L -q '-#{domain}' -c '#{collection}'`
-    .chomp.split("\n").map { |i| i.split("\t") }
+        .chomp.split("\n").map { |i| i.split("\t") }
 ]
 l_dom = n_dom.keys
 cnt_dom = {}
@@ -54,10 +54,10 @@ File.open(outlog, 'w') do |ofh|
   ofh.puts "! Contamination: #{q[:cnt].round(1)}%."
   if q[:multi] > 0
     ofh.puts "! Multiple copies: "
-    cnt_dom.each{ |k,v| ofh.puts "!   #{v} #{k}: #{n_dom[k]}." if v>1 }
+    cnt_dom.each { |k, v| ofh.puts "!   #{v} #{k}: #{n_dom[k]}." if v > 1 }
   end
   if q[:found] < cnt_dom.size
     ofh.puts "! Missing genes: "
-    cnt_dom.each{ |k,v| ofh.puts "!   #{k}: #{n_dom[k]}." if v==0 }
+    cnt_dom.each { |k, v| ofh.puts "!   #{k}: #{n_dom[k]}." if v == 0 }
   end
 end

data/utils/index_metadata.rb

@@ -13,16 +13,18 @@ db.execute 'create table metadata(' \
 
 def searchable(db, d, k, v)
   db.execute 'insert into metadata values(?,?,?)',
-    d.name, k.to_s, " #{v.to_s.downcase.gsub(/[^A-Za-z0-9\-]+/, ' ')} "
+             d.name, k.to_s, " #{v.to_s.downcase.gsub(/[^A-Za-z0-9\-]+/, ' ')} "
 end
 
 p.each_dataset do |d|
   next unless d.is_ref?
   next unless d.is_active?
+
   searchable(db, d, :name, d.name)
   d.metadata.each do |k, v|
     next if [:created, :updated].include? k
-    v = v.sorted_ranks.map{ |r| r[1] }.join(' ') if k == :tax
+
+    v = v.sorted_ranks.map { |r| r[1] }.join(' ') if k == :tax
     searchable(db, d, k, v)
   end
 end

data/utils/mytaxa_scan.rb

@@ -15,42 +15,45 @@ begin
 
   # Extract gene IDs
   ifh = faa =~ /\.gz/ ? Zlib::GzipReader.open(faa) : File.open(faa, 'r')
-  ids = ifh.each_line.grep(/^>/).map{|dl| dl.chomp.sub(/^>/,'').sub(/\s.*/,'')}
+  ids = ifh.each_line.grep(/^>/).map { |dl| dl.chomp.sub(/^>/, '').sub(/\s.*/, '') }
   ifh.close
-  tax = Hash[ids.map{|k| [k, "NA"]}]
+  tax = Hash[ids.map { |k| [k, "NA"] }]
 
   # Get MyTaxa distributions
   k, l = nil
   File.open(mytaxa).each do |ln|
     ln.chomp!
-    if $.%2 == 1
+    if $. % 2 == 1
       k, l = ln.split /\t/
     else
-      tax[k] = ln.gsub(/<[^>]+>/,"").gsub(/;/,"::")
+      tax[k] = ln.gsub(/<[^>]+>/, '').gsub(/;/, '::')
     end
   end
-  all_tax = tax.values.uniq.sort{|x,y| tax.values.count(y) <=> tax.values.count(x) }
+  all_tax = tax.values.uniq.sort do |x, y|
+    tax.values.count(y) <=> tax.values.count(x)
+  end
 
   # Estimate Windows and save gene IDs
-  fh = File.open(outdata + ".genes", "w")
+  fh = File.open(outdata + '.genes', 'w')
   c = []
-  c << all_tax.map{|t| tax.values.count(t) }
-  n_wins = (ids.size/winsize).ceil
-  (0 .. (n_wins-1)).each do |win|
-    k = ids[win*winsize, winsize]
+  c << all_tax.map { |t| tax.values.count(t) }
+  n_wins = (ids.size / winsize).ceil
+  (0..(n_wins - 1)).each do |win|
+    k = ids[win * winsize, winsize]
     win_t = tax.values_at(*k)
     fh.puts k.join("\t")
-    c << all_tax.map{|t| win_t.count(t)}
+    c << all_tax.map { |t| win_t.count(t) }
   end
-  p = c.map{|col| col.map{|cell| cell.to_f/col.inject(:+)}}
+  p = c.map { |col| col.map { |cell| cell.to_f / col.inject(:+) } }
   fh.close
 
   # Save window profiles
   fh = File.open(outdata, "w")
   fh.puts "# Data derived from #{mytaxa}, with #{winsize}-genes windows"
-  fh.puts "# " + (["Tax-label", "Genome"] + (1 .. n_wins).map{|i| "Win_#{i}"}).join("\t")
-  (0 .. (all_tax.size - 1)).each do |row|
-    fh.puts ([all_tax[row]] + p.map{|col| col[row]}).join "\t"
+  fh.puts '# ' + (['Tax-label', 'Genome'] +
+    (1..n_wins).map { |i| "Win_#{i}" }).join("\t")
+  (0..(all_tax.size - 1)).each do |row|
+    fh.puts ([all_tax[row]] + p.map { |col| col[row] }).join "\t"
   end
   fh.close
 rescue => err
@@ -58,4 +61,3 @@ rescue => err
   err.backtrace.each { |l| $stderr.puts l + "\n" }
   err
 end
-

data/utils/representatives.rb

@@ -19,7 +19,8 @@ end
 ani_spp = []
 File.open(pf, 'r') do |fh|
   fh.each_line do |ln|
-    next if $.==1 and ln.chomp == 'G' # <- Legacy check
+    next if $. == 1 and ln.chomp == 'G' # <- Legacy check
+
     ani_spp << ln.chomp.split(',')
   end
 end
@@ -32,10 +33,10 @@ ani_spp.each_with_index do |datasets, i|
     dr = d.result(:essential_genes) or next
     q = dr[:stats][:quality] or next
     if best.nil? or q > best[:q]
-      best = {d: d, q: q}
+      best = { d: d, q: q }
     end
   end
   raise "Unavailable statistics for any of:\n#{datasets}\n" if best.nil?
-  puts "ANIsp_#{i+1}\t#{best[:d].name}"
-end
 
+  puts "ANIsp_#{i + 1}\t#{best[:d].name}"
+end

data/utils/requirements.txt

@@ -1,6 +1,6 @@
 Software	Test exec	Website								Notes
 --------	---------	-------								-----
-Ruby		ruby		https://www.ruby-lang.org/					Required version: 2.1+
+Ruby		ruby		https://www.ruby-lang.org/					Required version: 2.3+
 Python		python		https://www.python.org/
 R		R		http://www.r-project.org/
 SQLite3		sqlite3		https://www.sqlite.org/

data/utils/subclade/base.rb

@@ -1,4 +1,3 @@
-
 require 'zlib'
 require 'miga'
 

data/utils/subclade/pipeline.rb

@@ -1,7 +1,5 @@
-
 # High-end pipelines for SubcladeRunner
 module MiGA::SubcladeRunner::Pipeline
-
   # Run species-level clusterings using ANI > 95% / AAI > 90%
   def cluster_species
     tasks = {
@@ -12,7 +10,7 @@ module MiGA::SubcladeRunner::Pipeline
       # Final output
       ogs_file = "miga-project.#{k}-clades"
       next if File.size? ogs_file
-      
+
       # Build ABC files
       abc_path = tmp_file("#{k}.abc")
       ofh = File.open(abc_path, 'w')
@@ -20,6 +18,7 @@ module MiGA::SubcladeRunner::Pipeline
       Zlib::GzipReader.open(metric_res.file_path(:matrix)) do |ifh|
         ifh.each_line do |ln|
           next if ln =~ /^metric\t/
+
           r = ln.chomp.split("\t")
           ofh.puts "G>#{r[1]}\tG>#{r[2]}\t#{r[3]}" if r[3].to_f >= par[1]
         end
@@ -55,7 +54,8 @@ module MiGA::SubcladeRunner::Pipeline
     ofh = File.open('miga-project.proposed-clades', 'w')
     File.open('miga-project.gsp-clades', 'r') do |ifh|
       ifh.each_line do |ln|
-        next if $.==1
+        next if $. == 1
+
         r = ln.chomp.split(',')
         ofh.puts r.join("\t") if r.size >= 5
       end
@@ -70,8 +70,9 @@ module MiGA::SubcladeRunner::Pipeline
     matrix = metric_res.file_path(:matrix)
     `Rscript '#{src}' '#{matrix}' miga-project '#{opts[:thr]}' \
       miga-project.ani95-medoids '#{opts[:run_clades] ? 'cluster' : 'empty'}'`
-    File.rename('miga-project.nwk',"miga-project.#{metric}.nwk") if
-          File.exist? 'miga-project.nwk'
+    if File.exist? 'miga-project.nwk'
+      File.rename('miga-project.nwk', "miga-project.#{metric}.nwk")
+    end
   end
 
   def compile

data/utils/subclade/runner.rb

@@ -1,33 +1,34 @@
-
 require_relative 'base.rb'
 require_relative 'temporal.rb'
 require_relative 'pipeline.rb'
 
 class MiGA::SubcladeRunner
-
   include MiGA::SubcladeRunner::Temporal
   include MiGA::SubcladeRunner::Pipeline
 
   attr_reader :project, :step, :opts, :home, :tmp
 
-  def initialize(project_path, step, opts_hash={})
+  def initialize(project_path, step, opts_hash = {})
     @opts = opts_hash
     @project = MiGA::Project.load(project_path) or
-          raise "No project at #{project_path}"
+      raise "No project at #{project_path}"
     @step = step.to_sym
-    clades_dir = File.expand_path('data/10.clades', project.path)
-    @home = File.expand_path(@step == :clade_finding ? '01.find' : '02.ani',
-          clades_dir)
+    @home = File.join(
+      File.join(project.path, 'data', '10.clades'),
+      @step == :clade_finding ? '01.find' : '02.ani'
+    )
     @opts[:thr] ||= ENV.fetch('CORES') { 2 }.to_i
     @opts[:run_clades] = !!@project.metadata.data.fetch(:run_clades) { true }
     @opts[:gsp_ani] = @project.metadata.data.fetch(:gsp_ani) { 95.0 }.to_f
     @opts[:gsp_aai] = @project.metadata.data.fetch(:gsp_aai) { 90.0 }.to_f
-    @opts[:gsp_metric] = @project.metadata.data.fetch(:gsp_metric){ 'ani' }.to_s
+    @opts[:gsp_metric] =
+      @project.metadata.data.fetch(:gsp_metric) { 'ani' }.to_s
   end
 
   # Launch the appropriate analysis
   def go!
     return if project.type == :metagenomes
+
     unless @project.dataset_names.any? { |i| @project.dataset(i).is_ref? }
       FileUtils.touch(File.expand_path('miga-project.empty', @home))
       return
@@ -54,5 +55,4 @@ class MiGA::SubcladeRunner
     subclades :ani
     compile
   end
-
 end