miga-base 0.3.1.5 → 0.3.1.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 36bdee987135a432485bcb9eb49895ad22c19870
-  data.tar.gz: 7409617014eb5adfb0adba8cdaa53c80db185db4
+  metadata.gz: 4ba91c88b5e9a25633e5633e344fc013b2ab0d6e
+  data.tar.gz: 24e7c2426c9ad4c86da246c3b4c76a94ed378aa9
 SHA512:
-  metadata.gz: f9000a92772143755f48e3c27e69cd9c42533cbd8946682239c3d3f9e8596342d4b5214d86d84363fa77dc8f8947915b764e3bd861431151a0dd86299db5fce3
-  data.tar.gz: 35b4eb28a73c23f601bfcac9777c25cf7116a69654f3b99a95e46118256041f2d0e2ec89dca379d97e46c6bae87fd5b64e481895e9f305573722905da6c3f22e
+  metadata.gz: bba625b0f7777a8aed26b0e3e55f16ada263c01e371f29ba7c89aa80f23afd5db9ae58c41abd1d0e37c6aaa17b924d00d487c6a09e222cd8d27b9e1394cd59d9
+  data.tar.gz: e906281a4ccf5b8f8505d6e7d5a342d44f16505c64602f1af4d65e3e09f456b21487c9c8273f84ec2c9cb2a009de13efb3b1fe20d65d879a2510abbbac7825b8

data/actions/doctor.rb

@@ -3,12 +3,27 @@
 # @package MiGA
 # @license Artistic-2.0
 
-o = {q:true, v:false}
+require "sqlite3"
+
+o = {q:true, ld:false,
+  db: true, dist: true, files: true, ess: true, mts: true, tax: true}
 OptionParser.new do |opt|
   opt_banner(opt)
   opt_object(opt, o, [:project])
-  opt.on("-v", "--verbose",
-    "Print additional information on advance."){ |v| o[:v]=v }
+  opt.on("-l", "--list-datasets",
+    "List all fixed datasets on advance."){ |v| o[:ld]=v }
+  opt.on("--ignore-databases",
+    "Do not check database files integrity."){ |v| o[:db]=!v }
+  opt.on("--ignore-distances",
+    "Do not check distance tables."){ |v| o[:dist]=!v }
+  opt.on("--ignore-files",
+    "Do not check for outdated files."){ |v| o[:files]=!v }
+  opt.on("--ignore-essential-genes",
+    "Do not check unarchived essential genes."){ |v| o[:ess]=!v }
+  opt.on("--ignore-mytaxa-scan",
+    "Do not check unarchived MyTaxa scan."){ |v| o[:mts]=!v }
+  opt.on("--ignore-taxonomy",
+    "Do not check taxonomy consistency."){ |v| o[:tax]=!v }
   opt_common(opt, o)
 end.parse!
 
@@ -19,60 +34,150 @@ $stderr.puts "Loading project" unless o[:q]
 p = MiGA::Project.load(o[:project])
 raise "Impossible to load project: #{o[:project]}" if p.nil?
 
+def check_sqlite3_database(db_file, metric)
+  begin
+    SQLite3::Database.new(db_file) do |conn|
+      conn.execute("select count(*) from #{metric}").first
+    end
+  rescue SQLite3::SQLException
+    yield
+  end
+end
+
+if o[:db]
+  $stderr.puts "o Checking databases integrity" unless o[:q]
+  p.each_dataset do |d|
+    [:distances, :taxonomy].each do |r_key|
+      r = d.result(r_key) or next
+      {haai_db: :aai, aai_db: :aai, ani_db: :ani}.each do |db_key, metric|
+        db_file = r.file_path(db_key) or next
+        check_sqlite3_database(db_file, metric) do
+          $stderr.puts(
+            "    > Removing #{db_key} #{r_key} table for #{d.name}.") if o[:ld]
+          [db_file, r.path(:done), r.path].each do |f|
+            File.unlink f if File.exist? f
+          end # each |f|
+        end # check_sqlite3_database
+      end # each |db_key, metric|
+    end # each |r_key|
+  end # each |d|
+end
+
 [:ani, :aai].each do |dist|
-  r = p.result("#{dist}_distances")
-  next if r.nil?
+  res = p.result("#{dist}_distances")
+  next if res.nil?
   $stderr.puts "o Checking #{dist} table for consistent datasets" unless o[:q]
-  ok = true
+  notok = {}
   fix = {}
-  Zlib::GzipReader.open(r.file_path(:matrix)) do |fh|
+  Zlib::GzipReader.open(res.file_path(:matrix)) do |fh|
+    lineno = 0
     fh.each_line do |ln|
-      next if $.==1
+      next if (lineno+=1)==1
       r = ln.split("\t")
-      if p.dataset(r[1]).nil? or p.dataset(r[2]).nil?
-        fix[r[2]] = true unless p.dataset(r[2]).nil?
-        fix[r[1]] = true unless p.dataset(r[1]).nil?
-        ok = false
+      if [1,2].map{ |i| p.dataset(r[i]).nil? }.any?
+        [1,2].each do |i|
+          if p.dataset(r[i]).nil?
+            notok[r[i]] = true
+          else
+            fix[r[i]] = true
+          end
+        end
       end
     end
   end
   
   $stderr.puts "  - Fixing #{fix.size} datasets" unless fix.empty? or o[:q]
   fix.keys.each do |d_n|
-    $stderr.puts "    > Fixing #{d_n}." if o[:v]
+    $stderr.puts "    > Fixing #{d_n}." if o[:ld]
     p.dataset(d_n).cleanup_distances!
   end
   
-  unless ok
-    $stderr.puts "  - Removing tables, recompute" unless o[:q]
-    r.remove!
+  unless notok.empty?
+    unless o[:q]
+      $stderr.puts "  - Unregistered datasets detected: "
+      if notok.size < 3
+        $stderr.puts "    - #{notok.keys.join(", ")}"
+      else
+        $stderr.puts "    - #{notok.keys.first} and other #{notok.size-1}"
+      end
+      $stderr.puts "  - Removing tables, recompute"
+    end
+    res.remove!
   end
-end
+end if o[:dist]
 
-$stderr.puts "o Looking for outdated files in results" unless o[:q]
-p.each_dataset do |d|
-  d.each_result do |r_k, r|
-    ok = true
-    r.each_file do |_f_sym, _f_rel, f_abs|
-      unless File.exist? f_abs
-        ok = false
-        break
+if o[:files]
+  $stderr.puts "o Looking for outdated files in results" unless o[:q]
+  p.each_dataset do |d|
+    d.each_result do |r_k, r|
+      ok = true
+      r.each_file do |_f_sym, _f_rel, f_abs|
+        unless File.exist? f_abs
+          ok = false
+          break
+        end
+      end
+      unless ok
+        $stderr.puts "    > Registering again #{d.name}:#{r_k}" if o[:ld]
+        d.add_result(r_k, true, force:true)
       end
     end
-    unless ok
-      $stderr.puts "  - Registering again #{d.name}:#{r_k}" if o[:v]
-      d.add_result(r_k, true, force:true)
+  end
+end
+
+if o[:ess]
+  $stderr.puts "o Looking for unarchived essential genes." unless o[:q]
+  p.each_dataset do |d|
+    res = d.result(:essential_genes)
+    next if res.nil?
+    dir = res.file_path(:collection)
+    if dir.nil?
+      $stderr.puts "    > Removing #{d.name}:essential_genes" if o[:ld]
+      res.remove!
+      next
+    end
+    unless Dir["#{dir}/*.faa"].empty?
+      $stderr.puts "    > Fixing #{d.name}." if o[:ld]
+      cmdo = `cd '#{dir}' && tar -zcf proteins.tar.gz *.faa && rm *.faa`.chomp
+      warn cmdo unless cmdo.empty?
     end
   end
 end
 
-#$stderr.puts "o Looking for unarchived essential genes." unless o[:q]
-#p.each_dataset do |d|
-#  TODO: Check unarchived protein files
-#end
+if o[:mts]
+  $stderr.puts "o Looking for unarchived MyTaxa Scan runs." unless o[:q]
+  p.each_dataset do |d|
+    res = d.result(:mytaxa_scan)
+    next if res.nil?
+    dir = res.file_path(:regions)
+    fix = false
+    unless dir.nil?
+      if Dir.exist? dir
+        cmdo = `cd '#{dir}/..' \
+              && tar -zcf '#{d.name}.reg.tar.gz' '#{d.name}.reg' \
+              && rm -r '#{d.name}.reg'`.chomp
+        warn cmdo unless cmdo.empty?
+      end
+      fix = true
+    end
+    %w[blast mytaxain wintax gene_ids region_ids].each do |ext|
+      file = res.file_path(ext.to_sym)
+      unless file.nil?
+        FileUtils.rm(file) if File.exist? file
+        fix = true
+      end
+    end
+    if fix
+      $stderr.puts "    > Fixing #{d.name}." if o[:ld]
+      d.add_result(:mytaxa_scan, true, force: true)
+    end
+  end
+end
 
-#$stderr.puts "o Checking for taxonomy/distances consistency" unless o[:q]
-# TODO: Find 95%ANI clusters with entries from different species
+if o[:tax]
+  #$stderr.puts "o Checking for taxonomy/distances consistency" unless o[:q]
+  # TODO: Find 95%ANI clusters with entries from different species
+end
 
 $stderr.puts "Done" unless o[:q]
 

data/actions/get.rb

@@ -30,8 +30,6 @@ OptionParser.new do |opt|
     "If set, ignores datasets that already exist."){ |v| o[:ignore_dup]=v }
   opt.on("-d", "--description STRING",
     "Description of the dataset."){ |v| o[:description]=v }
-  opt.on("-u", "--user STRING",
-    "Owner of the dataset."){ |v| o[:user]=v }
   opt.on("-c", "--comments STRING",
     "Comments on the dataset."){ |v| o[:comments]=v }
   opt_common(opt, o)
@@ -68,7 +66,7 @@ glob.each do |o_i|
   raise "Impossible to load project: #{o_i[:project]}" if p.nil?
 
   next if o_i[:ignore_dup] and not p.dataset(o_i[:dataset]).nil?
-  
+
   $stderr.puts "Locating remote dataset." unless o_i[:q]
   rd = MiGA::RemoteDataset.new(o_i[:ids], o_i[:db], o_i[:universe])
 

data/actions/ncbi_get.rb

@@ -0,0 +1,151 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/remote_dataset'
+
+o = {q:true, query:false, unlink:false,
+      reference: false, ignore_plasmids: false,
+      complete:false, chromosome:false,
+      scaffold:false, contig:false,}
+OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project])
+  opt.on("-T", "--taxon STRING",
+        "(Mandatory unless --reference) Name of the taxon (e.g., a species binomial)."
+        ){ |v| o[:taxon]=v }
+  opt.on("--reference",
+        "Download all reference genomes (ignores -T)."){ |v| o[:reference]=v }
+  opt.on("--ref-no-plasmids",
+        "If passed, ignores plasmids (only for --reference)."
+        ){ |v| o[:ignore_plasmids]=v }
+  opt.on("--complete", "Download complete genomes."){ |v| o[:complete]=v }
+  opt.on("--chromosome", "Download complete chromosomes."){ |v| o[:chromosome]=v }
+  opt.on("--scaffold", "Download genomes in scaffolds."){ |v| o[:scaffold]=v }
+  opt.on("--contig", "Download genomes in contigs."){ |v| o[:contig]=v }
+  opt.on("--all", "Download all genomes (in any status).") do
+    o[:complete] = true
+    o[:chromosome] = true
+    o[:scaffold] = true
+    o[:contig] = true
+  end
+  opt.on("-q", "--query",
+        "If set, the datasets are registered as queries, not reference datasets."
+        ){ |v| o[:query]=v }
+  opt.on("-u", "--unlink",
+        "If set, unlinks all datasets in the project missing from the download list."
+        ){ |v| o[:unlink]=v }
+  opt.on("-R", "--remote-list PATH",
+        "Path to an output file with the list of all datasets listed remotely."
+        ){ |v| o[:remote_list]=v }
+  opt_common(opt, o)
+end.parse!
+
+opt_require(o, project: "-P")
+opt_require(o, taxon: "-T") unless o[:reference]
+unless %w[reference complete chromosome scaffold contig].any?{ |i| o[i.to_sym] }
+  raise "No action requested. Pick at least one type of genome"
+end
+
+##=> Main <=
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+d = []
+ds = {}
+downloaded = 0
+
+def get_list(taxon, status)
+  url_base = "https://www.ncbi.nlm.nih.gov/genomes/Genome2BE/genome2srv.cgi?"
+  url_param = if status==:reference
+    { action: "refgenomes", download: "on" }
+  else
+    { action: "download", report: "proks", group: "-- All Prokaryotes --",
+          subgroup: "-- All Prokaryotes --", orgn: "#{taxon}[orgn]",
+          status: status }
+  end
+  url = url_base + URI.encode_www_form(url_param)
+  response = RestClient::Request.execute(method: :get, url:url, timeout:600)
+  unless response.code == 200
+    raise "Unable to reach NCBI, error code #{response.code}."
+  end
+  response.to_s
+end
+
+# Download IDs with reference status
+if o[:reference]
+  $stderr.puts "Downloading reference genomes" unless o[:q]
+  lineno = 0
+  get_list(nil, :reference).each_line do |ln|
+    next if (lineno+=1)==1
+    r = ln.chomp.split("\t")
+    next if r[3].nil? or r[3].empty?
+    ids = r[3].split(",")
+    ids += r[5].split(",") unless o[:ignore_plasmids] or r[5].empty?
+    n = r[2].miga_name
+    ds[n] = {ids: ids, md: {type: :genome}, db: :nuccore, universe: :ncbi}
+  end
+end
+
+# Download IDs with complete or chromosome status
+if o[:complete] or o[:chromosome]
+  status = (o[:complete] and o[:chromosome] ? "50|40" : o[:complete] ? "50" : "40")
+  $stderr.puts "Downloading complete/chromosome genomes" unless o[:q]
+  lineno = 0
+  get_list(o[:taxon], status).each_line do |ln|
+    next if (lineno+=1)==1
+    r = ln.chomp.split("\t")
+    next if r[10].nil? or r[10].empty?
+    ids = r[10].gsub(/[^:;]*:/,"").gsub(/\/[^\/;]*/,"").split(";")
+    n = (r[0] + "_" + ids[0]).miga_name
+    ds[n] = {ids: ids, md: {type: :genome}, db: :nuccore, universe: :ncbi}
+  end
+end
+
+# Download IDs with scaffold or contig status
+if o[:scaffold] or o[:contig]
+  status = (o[:scaffold] and o[:contig] ? "30|20" : o[:scaffold] ? "30" : "20")
+  $stderr.puts "Downloading scaffold/contig genomes" unless o[:q]
+  lineno = 0
+  get_list(o[:taxon], status).each_line do |ln|
+    next if (lineno+=1)==1
+    r = ln.chomp.split("\t")
+    next if r[7].nil? or r[7].empty?
+    next if r[19].nil? or r[19].empty?
+    asm = r[7].gsub(/[^:;]*:/,"").gsub(/\/[^\/;]*/,"").gsub(/\s/,"")
+    ids = r[19].gsub(/\s/, "").split(";").map{ |i| i + "/" + File.basename(i) + "_genomic.fna.gz" }
+    n = (r[0] + "_" + asm).miga_name
+    comm = "Assembly: #{asm}"
+    ds[n] = {ids: ids, md: {type: :genome, comments: comm}, db: :assembly_gz, universe: :web}
+  end
+end
+
+# Download entries
+$stderr.puts "Downloading #{ds.size} #{ds.size>1 ? "entries" : "entry"}." unless o[:q]
+ds.each do |name,body|
+  d << name
+  puts name
+  next unless p.dataset(name).nil?
+  $stderr.puts "  Locating remote dataset." unless o[:q]
+  rd = MiGA::RemoteDataset.new(body[:ids], body[:db], body[:universe])
+  $stderr.puts "  Creating dataset." unless o[:q]
+  rd.save_to(p, name, !o[:query], body[:md])
+  p.add_dataset(name)
+  downloaded += 1
+end
+
+# Finalize
+$stderr.puts "Datasets listed: #{d.size}" unless o[:q]
+$stderr.puts "Datasets downloaded: #{downloaded}" unless o[:q]
+unless o[:remote_list].nil?
+  File.open(o[:remote_list], 'w') do |fh|
+    d.each { |i| fh.puts i }
+  end
+end
+if o[:unlink]
+  unlink = p.dataset_names - d
+  unlink.each { |i| p.unlink_dataset(i).remove! }
+  $stderr.puts "Datasets unlinked: #{unlink.size}" unless o[:q]
+end
+

data/bin/miga

@@ -19,6 +19,7 @@ $task_desc = {
   # Datasets
   add: "Creates an empty dataset in a pre-existing MiGA project.",
   get: "Downloads a dataset from public databases into a MiGA project.",
+  ncbi_get: "Downloads all genomes in a taxon or RefSeq from NCBI into a MiGA project.",
   rm: "Removes a dataset from an MiGA project.",
   find: "Finds unregistered datasets based on result files.",
   ln: "Link datasets (including results) from one project to another.",
@@ -102,7 +103,7 @@ def opt_object(opt, o, what=[:project, :dataset])
       what.include? :dataset_type_req
   opt.on("-t", "--type STRING",
     (what.include?(:project_type_req) ? "(Mandatory) " : "") +
-    "Type of project. Recognized types include:", 
+    "Type of project. Recognized types include:",
     *MiGA::Project.KNOWN_TYPES.map{ |k,v| "~ #{k}: #{v[:description]}"}
     ){ |v| o[:type]=v.to_sym } if what.include? :project_type or
       what.include? :project_type_req
@@ -228,4 +229,3 @@ generic options:
 
 HELP
 end
-

data/lib/miga/common.rb

@@ -1,21 +1,23 @@
 # @package MiGA
 # @license Artistic-2.0
 
-require "miga/version"
-require "json"
-require "tempfile"
-require "zlib"
+require 'miga/version'
+require 'json'
+require 'tempfile'
+require 'zlib'
 
 ##
 # Generic class used to handle system-wide information and methods, and parent
 # of all other MiGA::* classes.
 class MiGA::MiGA
   
-  ENV["MIGA_HOME"] ||= ENV["HOME"]
+  ENV['MIGA_HOME'] ||= ENV['HOME']
 
   ##
   # Root path to MiGA (as estimated from the location of the current file).
-  def self.root_path ; File.expand_path("../../..", __FILE__) ; end
+  def self.root_path
+    File.expand_path('../../..', __FILE__)
+  end
 
   ##
   # Should debugging information be reported?
@@ -48,32 +50,32 @@ class MiGA::MiGA
 
   ##
   # Send debug message.
-  def self.DEBUG *args
+  def self.DEBUG(*args)
     $stderr.puts(*args) if @@DEBUG
-    $stderr.puts caller.map{|v| v.gsub(/^/,"    ")}.join("\n") if
-      @@DEBUG_TRACE
+    if @@DEBUG_TRACE
+      $stderr.puts caller.map{ |v| v.gsub(/^/,'     ') }.join("\n")
+    end
   end
 
   ##
   # Has MiGA been initialized?
   def self.initialized?
-    File.exist?(File.expand_path(".miga_rc", ENV["MIGA_HOME"])) and
-      File.exist?(File.expand_path(".miga_daemon.json", ENV["MIGA_HOME"]))
+    File.exist?(File.expand_path('.miga_rc', ENV['MIGA_HOME'])) and
+      File.exist?(File.expand_path('.miga_daemon.json', ENV['MIGA_HOME']))
   end
 
   ##
   # Tabulates an +values+, and Array of Arrays, all with the same number of
   # entries as +header+. Returns an Array of String, one per line.
   def self.tabulate(header, values)
-    fields = [header.map{ |h| h.to_s }]
-    fields << fields.first.map{ |h| h.gsub(/\S/, "-") }
-    fields += values.map{ |row| row.map{ |cell| cell.nil? ? "?" : cell.to_s } }
-    clen = fields.map{ |row|
-      row.map{ |cell| cell.length } }.transpose.map{ |col| col.max }
+    fields = [header.map(&:to_s)]
+    fields << fields.first.map{ |h| h.gsub(/\S/, '-') }
+    fields += values.map{ |row| row.map{ |cell| cell.nil? ? '?' : cell.to_s } }
+    clen = fields.map{ |row| row.map(&:length) }.transpose.map(&:max)
     fields.map do |row|
       (0 .. clen.size-1).map do |col_n|
         col_n==0 ? row[col_n].rjust(clen[col_n]) : row[col_n].ljust(clen[col_n])
-      end.join("  ")
+      end.join('  ')
     end
   end
 
@@ -82,25 +84,25 @@ class MiGA::MiGA
   def self.clean_fasta_file(file)
     tmp_fh = nil
     begin
-      if (file =~ /\.gz/)
-        tmp_path = Tempfile.new("MiGA.gz").tap{ |i| i.close }.path
+      if file =~ /\.gz/
+        tmp_path = Tempfile.new('MiGA.gz').tap(&:close).path
         tmp_fh = Zlib::GzipWriter.open(tmp_path)
         fh = Zlib::GzipReader.open(file)
       else
-        tmp_fh = Tempfile.new("MiGA")
+        tmp_fh = Tempfile.new('MiGA')
         tmp_path = tmp_fh.path
-        fh = File.open(file, "r")
+        fh = File.open(file, 'r')
       end
-      buffer = ""
+      buffer = ''
       fh.each_line do |ln|
         ln.chomp!
         if ln =~ /^>\s*(\S+)(.*)/
           (id, df) = [$1, $2]
           tmp_fh.print buffer.wrap_width(80)
-          buffer = ""
+          buffer = ''
           tmp_fh.puts ">#{id.gsub(/[^A-Za-z0-9_\|\.]/, "_")}#{df}"
         else
-          buffer << ln.gsub(/[^A-Za-z\.\-]/, "")
+          buffer << ln.gsub(/[^A-Za-z\.\-]/, '')
         end
       end
       tmp_fh.print buffer.wrap_width(80)
@@ -124,7 +126,7 @@ class MiGA::MiGA
   # - +:n50+: If true, it also returns the N50 and the median (in bp).
   # - +gc+: If true, it also returns the G+C content (in %).
   def self.seqs_length(file, format, opts={})
-    fh = (file =~ /\.gz/) ? Zlib::GzipReader.open(file) : File.open(file, "r")
+    fh = (file =~ /\.gz/) ? Zlib::GzipReader.open(file) : File.open(file, 'r')
     l = []
     gc = 0
     i = 0 # <- Zlib::GzipReader doesn't set $.
@@ -154,12 +156,11 @@ class MiGA::MiGA
         break if pos >= thr
       end
       o[:med] = o[:n].even? ?
-        0.5*( l[o[:n]/2-1,2].inject(:+) ) : l[(o[:n]-1)/2]
+        0.5*l[o[:n]/2-1,2].inject(:+) : l[(o[:n]-1)/2]
     end
     o
   end
-  
-  
+
   ##
   # Path to a script to be executed for +task+. Supported +opts+ are:
   # - +:miga+ Path to the MiGA home to use. If not passed, the home of the
@@ -178,17 +179,15 @@ class MiGA::MiGA
     File.expand_path("scripts/#{task}.bash", opts[:miga])
   end
 
-
   ##
   # Check if the result files exist with +base+ name (String) followed by the
   # +ext+ values (Array of String).
   def result_files_exist?(base, ext)
-    ext = [ext] unless ext.kind_of? Array
+    ext = [ext] unless ext.is_a? Array
     ext.all? do |f|
       File.exist?(base + f) or File.exist?("#{base}#{f}.gz")
     end
   end
-
 end
 
 ##
@@ -212,29 +211,36 @@ class File
       raise "Unknown transfer method: #{method}."
     end
   end
-  
 end
 
 ##
 # MiGA extensions to the String class.
 class String
-  
+
   ##
   # Replace any character not allowed in a MiGA name for underscore (_). This
   # results in a MiGA-compliant name EXCEPT for empty strings, that results in
   # empty strings.
-  def miga_name ; gsub(/[^A-Za-z0-9_]/, "_") ; end
+  def miga_name
+    gsub(/[^A-Za-z0-9_]/, '_')
+  end
 
   ##
   # Is the string a MiGA-compliant name?
-  def miga_name? ; not(self !~ /^[A-Za-z0-9_]+$/) ; end
+  def miga_name?
+    !(self !~ /^[A-Za-z0-9_]+$/)
+  end
 
   ##
   # Replace underscores by spaces or dots (depending on context).
-  def unmiga_name ; gsub(/_(str|sp|subsp|pv)__/,"_\\1._").tr("_", " ") ; end
-  
+  def unmiga_name
+    gsub(/_(str|sp|subsp|pv)__/,"_\\1._").tr('_', ' ')
+  end
+
   ##
   # Wraps the string with fixed Integer +width+.
-  def wrap_width(width) ; gsub(/([^\n\r]{1,#{width}})/,"\\1\n") ; end
-
+  def wrap_width(width)
+    gsub(/([^\n\r]{1,#{width}})/,"\\1\n")
+  end
 end
+

data/lib/miga/daemon.rb

@@ -191,7 +191,7 @@ class MiGA::Daemon < MiGA::MiGA
     log_dir = File.expand_path("daemon/#{job}", project.path)
     Dir.mkdir(log_dir) unless Dir.exist? log_dir
     task_name = "#{project.metadata[:name][0..9]}:#{job}:#{ds_name}"
-    to_run = {ds: ds, job: job, task_name: task_name,
+    to_run = {ds: ds, ds_name: ds_name, job: job, task_name: task_name,
       cmd: sprintf(runopts(:cmd),
         # 1: script
         MiGA::MiGA.script_path(job, miga:vars['MIGA'], project:project),

data/lib/miga/dataset/result.rb

@@ -42,7 +42,9 @@ module MiGA::Dataset::Result
     dir = @@RESULT_DIRS[result_type]
     return nil if dir.nil?
     base = File.expand_path("data/#{dir}/#{name}", project.path)
-    unless opts[:force]
+    if opts[:force]
+      FileUtils.rm("#{base}.json") if File.exist?("#{base}.json")
+    else
       r_pre = MiGA::Result.load("#{base}.json")
       return r_pre if (r_pre.nil? and not save) or not r_pre.nil?
     end
@@ -204,7 +206,7 @@ module MiGA::Dataset::Result
     ##
     # Add result type +:cds+ at +base+. Hash +opts+ supports +is_clean: Boolean+
     def add_result_cds(base, opts)
-      return nil unless result_files_exist?(base, %w[.faa .fna])
+      return nil unless result_files_exist?(base, %w[.faa])
       r = MiGA::Result.new("#{base}.json")
       r = add_files_to_ds_result(r, name, proteins:".faa", genes:".fna",
         gff2:".gff2", gff3:".gff3", tab:".tab")
@@ -266,13 +268,15 @@ module MiGA::Dataset::Result
     def add_result_mytaxa_scan(base, _opts)
       if is_nonmulti?
         return nil unless
-          result_files_exist?(base, %w[.pdf .wintax .mytaxa .reg]) or
-          result_files_exist?(base, ".nomytaxa.txt")
+          result_files_exist?(base, %w[.pdf .mytaxa]) or
+          result_files_exist?(base, '.nomytaxa.txt')
         r = MiGA::Result.new("#{base}.json")
-        add_files_to_ds_result(r, name, mytaxa:".mytaxa", wintax:".wintax",
-          blast:".blast", mytaxain:".mytaxain", report:".pdf", regions:".reg",
-          gene_ids:".wintax.genes", region_ids:".wintax.regions",
-          nomytaxa:".nomytaxa.txt")
+        add_files_to_ds_result(r, name, nomytaxa: '.nomytaxa.txt',
+          mytaxa: '.mytaxa', report: '.pdf', regions_archive: '.reg.tar',
+          # Intermediate / Deprecated
+          blast: '.blast', mytaxain: '.mytaxain', wintax: '.wintax',
+          gene_ids: '.wintax.genes', region_ids: '.wintax.regions',
+          regions: '.reg')
       else
         MiGA::Result.new("#{base}.json")
       end

data/lib/miga/result.rb

@@ -21,10 +21,6 @@ class MiGA::Result < MiGA::MiGA
 
   # Instance-level
 
-  ##
-  # Path to the JSON file describing the result.
-  attr_reader :path
-  
   ##
   # Hash with the result metadata.
   attr_reader :data
@@ -48,6 +44,22 @@ class MiGA::Result < MiGA::MiGA
   # Register the result as cleaned.
   def clean! ; self[:clean] = true ; end
 
+  ##
+  # Path to the standard files of the result. +which+ must be one of:
+  # - :json (default) : JSON file describing the result.
+  # - :start : File with the date when the processing started.
+  # - :done : File with the date when the processing ended.
+  def path(which=:json)
+    case which.to_sym
+    when :json
+      @path
+    when :start
+      @path.sub(/\.json$/, ".start")
+    when :done
+      @path.sub(/\.json$/, ".done")
+    end
+  end
+
   ##
   # Directory containing the result.
   def dir

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.3, 1, 5]
+  VERSION = [0.3, 1, 6]
   
   ##
   # Nickname for the current major.minor version.

data/scripts/essential_genes.bash

@@ -37,7 +37,10 @@ else
 fi
 
 # Reduce files
-( cd "${DATASET}.ess" && tar -zcf proteins.tar.gz *.faa && rm *.faa )
+( cd "${DATASET}.ess" \
+    && exists *.faa \
+    && tar -zcf proteins.tar.gz *.faa \
+    && rm *.faa )
 
 # Finalize
 miga date > "$DATASET.done"

data/scripts/mytaxa_scan.bash

@@ -75,14 +75,17 @@ else
         FastA.filter.pl -q "$DATASET.reg/$i.ids" \
           "../../../06.cds/$DATASET.faa" > "$DATASET.reg/$i.faa"
       done
+      # Archive regions
+      tar zcf "$DATASET.reg.tar.gz" "$DATASET.reg"
+      rm -r "$DATASET.reg"
     fi
 
     # Clean
-    [[ -e "$DATASET.daa" ]] && rm "$DATASET.daa"
-    [[ -s "$DATASET.blast" && ! -s "$DATASET.blast.gz" ]] \
-      && gzip -9 -f "$DATASET.blast"
-    [[ -s "$DATASET.mytaxain" && ! -s "$DATASET.mytaxain.gz" ]] \
-      && gzip -9 -f "$DATASET.mytaxain"
+    for x in daa blast mytaxain wintax wintax.genes wintax.regions ; do
+      [[ -e "$DATASET.$x" ]] && rm "$DATASET.$x"
+    done
+    [[ -s "$DATASET.mytaxa" && ! -s "$DATASET.mytaxa.gz" ]] \
+      && gzip -9 -f "$DATASET.mytaxa"
   fi
 
 fi

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.3.1.5
+  version: 0.3.1.6
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-31 00:00:00.000000000 Z
+date: 2018-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -130,6 +130,7 @@ files:
 - actions/init.rb
 - actions/ln.rb
 - actions/ls.rb
+- actions/ncbi_get.rb
 - actions/new.rb
 - actions/plugins.rb
 - actions/rm.rb
@@ -196,9 +197,6 @@ files:
 - utils/adapters.fa
 - utils/arch-ess-genes.rb
 - utils/core-pan-plot.R
-- utils/distances.rb
-- utils/distances/functions.rb
-- utils/distances/ref-nomulti.rb
 - utils/enveomics/Examples/aai-matrix.bash
 - utils/enveomics/Examples/ani-matrix.bash
 - utils/enveomics/Examples/essential-phylogeny.bash

data/utils/distances/functions.rb

@@ -1,58 +0,0 @@
-
-require 'sqlite3'
-
-$opts = {}
-if ENV["MIGA_AAI_SAVE_RBM"].nil?
-  $opts[:aai_save_rbm] = $project.is_clade? ? "save-rbm" : "no-save-rbm"
-else
-  $opts[:aai_save_rbm] = ENV["MIGA_AAI_SAVE_RBM"]
-end
-$opts[:thr] = ENV["CORES"].nil? ? 2 : ENV["CORES"].to_i
-
-def ani(f1, f2, db, opts={})
-  opts = $opts.merge(opts)
-  v = `ani.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" --name1 "#{ds_name f1}" --name2 "#{ds_name f2}" \
-        -t "#{opts[:thr]}" -a --no-save-regions --no-save-rbm --lookup-first`
-  v.nil? or v.empty? ? 0 : v.to_f
-end
-
-def make_empty_aai_db(db)
-  SQLite3::Database.new(db) do |conn|
-    conn.execute "create table if not exists aai(" +
-        "seq1 varchar(256), seq2 varchar(256), " +
-        "aai float, sd float, n int omega int" +
-      ")"
-  end unless File.size?(db)
-end
-
-def aai(f1, f2, db, opts={})
-  opts = $opts.merge(opts)
-  v = `aai.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" --name1 "#{ds_name f1}" --name2 "#{ds_name f2}" \
-        -t "#{opts[:thr]}" -a --lookup-first "--#{opts[:aai_save_rbm]}"`.chomp
-  v.nil? or v.empty? ? 0 : v.to_f
-end
-
-def haai(f1, f2, db, aai_db, opts={})
-  opts = $opts.merge(opts)
-  haai = aai(f1, f2, db, aai_save_rbm: "no-save-rbm")
-  return 0 if haai.nil? or haai == 0 or haai > 90.0
-  aai = 100.0 - Math.exp(2.435076 + 0.4275193*Math.log(100.0-haai))
-  make_empty_aai_db(aai_db)
-  SQLite3::Database.new(db) do |conn|
-    conn.execute "insert into aai values(?, ?, ?, 0, 0, 0)",
-          [ds_name(f1), ds_name(f2), aai]
-  end
-  aai
-end
-
-def haai_or_aai(f1_h, f2_h, db_h, f1, f2, db, opts={})
-  haai=haai(f1_h, f2_h, db_h, db, opts)
-  aai = aai(f1, f2, db, opts) if aai.nil? or aai.zero?
-  aai
-end
-
-def val_from_db(n1, n2, db, metric)
-  SQLite3::Database.new(db) do |conn|
-    return conn.execute("select #{metric} from #{metric} where seq1=? and seq2=?", [n1, n2]).first.first
-  end if File.size? db
-end

data/utils/distances/ref-nomulti.rb

@@ -1,2 +0,0 @@
-
-require_relative 'functions.rb'

data/utils/distances.rb

@@ -1,16 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'miga'
-
-$project = MiGA::Project.load(ARGV.shift)
-$dataset = $project.dataset(ARGV.shift)
-opts = Hash[ARGV]
-
-exit(0) if dataset.is_multi?
-
-if dataset.is_ref?
-  require_relative 'distances/ref-nomulti.rb'
-else
-  require_relative 'distances/noref-nomulti.rb'
-end
-# TODO run_distances!!!