miga-base 0.7.13.1 → 0.7.16.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3065d52418e9cc788496eddd1ae050c91645bfe1c1cf55c18950cf764046d8fc
-  data.tar.gz: c4b5b51569ab2b213dcdc4e956f36adf455074aeae372eab90115d1c0fa1d3f7
+  metadata.gz: 2ee2f247a9f26ddbf2fcf0fc85bb0ed51d0d4980e2e40297449665a0befca102
+  data.tar.gz: 9d2ea26b71d226f0cbddebb214912a82b432886efb72e8bb6523b4b2f5aa588c
 SHA512:
-  metadata.gz: 25fa750649f05083834a71d84efa8fd036cd3590f0f311cb158da748f6d78ad387eba6351af24a7c2f485f7d97f4af9cc7fc2eca615ae483f8ee052c8a001806
-  data.tar.gz: bbe8c5e2bc8630e44fb195704a91b7d7a95401e2fee7933982a775d1c6e1a28868f773a60369c4ca087fe427349ffcc780515f528bff522fbb0cbf008c51f4ae
+  metadata.gz: 673a1f46b1e34e14b58a434be79ec92c2ed31e7225004bd3fe08daebdf1362603353b5dd8473de8923583ed9819b6fe54578a3e18ea33dd51a54e2fad0bb0755
+  data.tar.gz: 23e44862a13fa293ea72d2260fc9676e2438b4c0c1725c64863a23c144d9b8a27dfefa62da6234fc0e86a8e59ada90498801a5609baaf07357293807801ccf60

data/README.md

@@ -11,11 +11,12 @@
 
 For additional information on MiGA, visit:
 
-* [MiGA Online][miga-online]: The Microbial Genomes Atlas Online.
+* [MiGA Online][miga-online]: The Microbial Genomes Atlas Online
+* [MiGA@XSEDE][miga-at-xsede]: The MiGA@XSEDE Gateway
 * [MiGA users list][mailing-list]:
-  Forum to discuss with other users and developers.
-* [MiGA manual][manual]: The definitive guide to MiGA.
-* [MiGA API docs][rubydoc]: Inner-workings of the `miga-base` gem.
+  Forum to discuss with other users and developers
+* [MiGA manual][manual]: The definitive guide to MiGA
+* [MiGA API docs][rubydoc]: Inner-workings of the `miga-base` gem
 * [MiGA Web][miga-web]: MiGA on Rails!
 
 # For the impatient
@@ -39,7 +40,8 @@ You have two options:
 
 Developed and maintained by [Luis M. Rodriguez-R][lrr]. MiGA is the result of a
 collaboration between [Kostas Lab][kostas] at the Georgia Institute of
-Technology and [RDP][rdp] at Michigan State University.
+Technology, [Rodriguez-R Lab][lrr] at the University of Innsbruck,
+and [RDP][rdp] at Michigan State University.
 
 See also the [complete list of contributors](manual/part1/contributors.md).
 
@@ -47,7 +49,7 @@ See also the [complete list of contributors](manual/part1/contributors.md).
 
 See [LICENSE](LICENSE).
 
-[lrr]: http://lmrodriguezr.github.io/
+[lrr]: https://rodriguez-r.com/
 [mailing-list]: http://support.microbial-genomes.org/
 [manual]: https://manual.microbial-genomes.org/
 [rubydoc]: http://www.rubydoc.info/github/bio-miga/miga
@@ -55,5 +57,6 @@ See [LICENSE](LICENSE).
 [miga-web]: https://github.com/bio-miga/miga-web
 [miga-gui]: https://github.com/bio-miga/miga-gui
 [miga-online]: http://microbial-genomes.org/
+[miga-at-xsede]: https://xsede.microbial-genomes.org/
 [kostas]: http://enve-omics.gatech.edu/
 [rdp]: http://rdp.cme.msu.edu/

data/lib/miga/cli.rb

@@ -110,30 +110,9 @@ class MiGA::Cli < MiGA::MiGA
   end
 
   ##
-  # Reports the advance of a task at +step+ (String), the +n+ out of +total+.
-  # The advance is reported in powers of 1,024 if +bin+ is true, or powers of
-  # 1,000 otherwise.
-  # The report goes to $stderr iff --verborse
-  def advance(step, n = 0, total = nil, bin = true)
-    return unless self[:verbose]
-
-    adv = total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
-      ('%.1f%% (%s/%s)' % [100.0 * n / total,
-                           num_suffix(n, bin), num_suffix(total, bin)])
-    $stderr.print("[%s] %s %s    \r" % [Time.now, step, adv])
-  end
-
-  def num_suffix(n, bin = false)
-    p = ''
-    { T: 4, G: 3, M: 2, K: 1 }.each do |k, x|
-      v = (bin ? 1024 : 1e3)**x
-      if n > v
-        n = '%.1f' % (n / v)
-        p = k
-        break
-      end
-    end
-    "#{n}#{p}"
+  # Same as MiGA::MiGA#advance, but checks if the CLI is verbose
+  def advance(*par)
+    super(*par) if self[:verbose]
   end
 
   ##
@@ -189,8 +168,9 @@ class MiGA::Cli < MiGA::MiGA
   end
 
   ##
-  # Perform the task requested (see #task)
-  def launch
+  # Perform the task requested (see #task); if +abort_on_error+, abort on
+  # error
+  def launch(abort_on_error = false)
     begin
       raise "See `miga -h`" if action.nil?
 
@@ -199,6 +179,7 @@ class MiGA::Cli < MiGA::MiGA
       $stderr.puts "Exception: #{err}"
       $stderr.puts ''
       err.backtrace.each { |l| $stderr.puts "DEBUG: #{l}" }
+      abort if abort_on_error
       err
     end
   end

data/lib/miga/cli/action/browse.rb

@@ -15,12 +15,12 @@ class MiGA::Cli::Action::Browse < MiGA::Cli::Action
     p = cli.load_project
     create_empty_page(p)
     generate_project_page(p)
-    say 'Creating dataset pages'
+    cli.say 'Creating dataset pages'
     cli.load_project.each_dataset do |d|
       generate_dataset_page(p, d)
     end
     generate_datasets_index(p)
-    say "Open in your browser: #{File.join(p.path, 'index.html')}"
+    cli.say "Open in your browser: #{File.join(p.path, 'index.html')}"
   end
 
   private
@@ -28,7 +28,7 @@ class MiGA::Cli::Action::Browse < MiGA::Cli::Action
   ##
   # Create an empty page with necessary assets for project +p+
   def create_empty_page(p)
-    say 'Creating project page'
+    cli.say 'Creating project page'
     FileUtils.mkdir_p(browse_file(p, '.'))
     %w[favicon-32.png style.css].each do |i|
       FileUtils.cp(template_file(i), browse_file(p, i))
@@ -103,7 +103,7 @@ class MiGA::Cli::Action::Browse < MiGA::Cli::Action
   ##
   # Create pages for reference and query dataset indexes
   def generate_datasets_index(p)
-    say 'Creating index pages'
+    cli.say 'Creating index pages'
     data = format_dataset_index(p)
     data.each do |k, v|
       write_file(p, "#{k}_datasets.html") do

data/lib/miga/cli/action/classify_wf.rb

@@ -42,9 +42,7 @@ class MiGA::Cli::Action::ClassifyWf < MiGA::Cli::Action
         '--no-summaries',
         'Do not generate intermediate step summaries'
       ) { |v| cli[:summaries] = v }
-      opts_for_wf(
-        opt, 'Input genome assemblies (nucleotides, FastA)', qual: false
-      )
+      opts_for_wf(opt, 'Input genome assemblies (nucleotides, FastA)')
     end
   end
 

data/lib/miga/cli/action/daemon.rb

@@ -6,7 +6,7 @@ require 'miga/daemon'
 
 class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
   def parse_cli
-    cli.defaults = { daemon_opts: [] }
+    cli.defaults = { daemon_opts: [], show_log: false }
     cli.expect_operation = true
     cli.parse do |opt|
       opt.separator 'Available operations:'
@@ -45,6 +45,10 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
         '--json PATH',
         'Path to a custom daemon definition in json format'
       ) { |v| cli[:json] = v }
+      opt.on(
+        '--show-log',
+        'Display log on advance instead of the progress summary'
+      ) { |v| cli[:show_log] = v }
       cli.opt_common(opt)
 
       opt.separator 'Daemon options:'
@@ -73,6 +77,7 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
     d = MiGA::Daemon.new(p, cli[:json])
     dopts = %i[latency maxjobs nodelist ppn shutdown_when_done]
     dopts.each { |k| d.runopts(k, cli[k]) }
+    d.show_log! if cli[:show_log]
     d.daemon(cli.operation, cli[:daemon_opts])
   end
 end

data/lib/miga/cli/action/doctor.rb

@@ -37,6 +37,7 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   @@OPERATIONS = {
     status: ['status', 'Update metadata status of all datasets'],
     db: ['databases', 'Check integrity of database files'],
+    bidir: ['bidirectional', 'Check distances are bidirectional'],
     dist: ['distances', 'Check distance summary tables'],
     files: ['files', 'Check for outdated files'],
     cds: ['cds', 'Check for gzipped genes and proteins'],
@@ -85,6 +86,24 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
     cli.say
   end
 
+  ##
+  # Perform bidirectional operation with MiGA::Cli +cli+
+  def check_bidir(cli)
+    cli.say 'Checking that reference distances are bidirectional'
+    ref_ds = cli.load_project.each_dataset.select(&:ref?)
+    ref_names = ref_ds.map(&:name)
+    n, k = ref_ds.size, 0
+    ref_ds.each do |d|
+      cli.advance('Datasets:', k += 1, n, false)
+      saved = saved_targets(d)
+      next if saved.nil?
+
+      to_save = ref_names - saved
+      to_save.each { |k| save_bidirectional(cli.load_project.dataset(k), d) }
+    end
+    cli.say
+  end
+
   ##
   # Perform distances operation with MiGA::Cli +cli+
   def check_dist(cli)

data/lib/miga/cli/action/doctor/base.rb

@@ -99,4 +99,46 @@ module MiGA::Cli::Action::Doctor::Base
     cli.say '- Removing tables, recompute'
     res.remove!
   end
+
+  ##
+  # Returns all targets identified by AAI
+  def saved_targets(dataset)
+    # Return nil if distance or database are not retrievable
+    dist = dataset.result(:distances) or return
+    path = dist.file_path(:aai_db) or return
+
+    o = []
+    SQLite3::Database.new(path) do |conn|
+      o = conn.execute('select seq2 from aai').map(&:first)
+    end
+    o
+  end
+
+  ##
+  # Saves all the distance estimates in +a+ -> +b+ into the +b+ databases
+  # (as +b+ -> +a+), where both +a+ and +b+ are MiGA::Dataset objects
+  def save_bidirectional(a, b)
+    each_database_file(a) do |db_file, metric, result|
+      data = nil
+      SQLite3::Database.new(db_file) do |conn|
+        data =
+          conn.execute(
+            "select seq1, seq2, #{metric}, sd, n, omega " +
+            "from #{metric} where seq2 = ? limit 1", b.name
+          ).first
+      end
+      next if data.nil? || data.empty?
+
+      db_file_b = File.join(File.dirname(db_file), "#{b.name}.db")
+      next unless File.exist?(db_file_b)
+
+      data[0], data[1] = data[1], data[0]
+      SQLite3::Database.new(db_file_b) do |conn|
+        conn.execute(
+          "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
+          "values(?, ?, ?, ?, ?, ?)", data
+        )
+      end
+    end
+  end
 end

data/lib/miga/cli/action/quality_wf.rb

@@ -9,16 +9,13 @@ class MiGA::Cli::Action::QualityWf < MiGA::Cli::Action
 
   def parse_cli
     default_opts_for_wf
-    cli.defaults = { mytaxa: false }
+    cli.defaults = { mytaxa: false, min_qual: 'no' }
     cli.parse do |opt|
       opt.on(
         '-m', '--mytaxa-scan',
         'Perform MyTaxa scan analysis'
       ) { |v| cli[:mytaxa] = v }
-      opts_for_wf(
-        opt, 'Input genome assemblies (nucleotides, FastA)',
-        qual: false
-      )
+      opts_for_wf(opt, 'Input genome assemblies (nucleotides, FastA)')
     end
   end
 

data/lib/miga/cli/action/wf.rb

@@ -9,7 +9,7 @@ module MiGA::Cli::Action::Wf
     cli.defaults = {
       clean: false, regexp: MiGA::Cli.FILE_REGEXP,
       project_type: :genomes, dataset_type: :popgenome,
-      ncbi_draft: true
+      ncbi_draft: true, min_qual: 25.0
     }
   end
 
@@ -42,10 +42,10 @@ module MiGA::Cli::Action::Wf
     end
     if params[:qual]
       opt.on(
-        '--min-qual FLOAT', Float,
+        '--min-qual FLOAT',
         'Minimum genome quality to include in analysis',
-        'By default: 50.0'
-      ) { |v| cli[:min_qual] = v }
+        "By default: #{cli[:min_qual]}"
+      ) { |v| cli[:min_qual] = v == 'no' ? v : v.to_f }
     end
     if params[:cleanup]
       opt.on(

data/lib/miga/common.rb

@@ -6,6 +6,7 @@ require 'miga/json'
 require 'miga/common/base'
 require 'miga/common/path'
 require 'miga/common/format'
+require 'stringio'
 
 ##
 # Generic class used to handle system-wide information and methods, and parent
@@ -38,10 +39,42 @@ class MiGA::MiGA
   ##
   # Print +par+ ensuring new line at the end.
   # Date/time-stamp each line.
-  # If the first parameter is +IO+, the output is sent there,
+  # If the first parameter is +IO+ or +StringIO+ the output is sent there,
   # otherwise it's sent to +$stderr+
   def say(*par)
-    io = par.first.is_a?(IO) ? par.shift : $stderr
+    io = like_io?(par.first) ? par.shift : $stderr
     io.puts(*par.map { |i| "[#{Time.now}] #{i}" })
   end
+
+  ##
+  # Reports the advance of a task at +step+ (String), the +n+ out of +total+.
+  # The advance is reported in powers of 1,024 if +bin+ is true, or powers of
+  # 1,000 otherwise.
+  # The report goes to $stderr iff --verborse
+  def advance(step, n = 0, total = nil, bin = true)
+    adv = total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
+      ('%.1f%% (%s/%s)' % [100.0 * n / total,
+                           num_suffix(n, bin), num_suffix(total, bin)])
+    $stderr.print("[%s] %s %s    \r" % [Time.now, step, adv])
+  end
+
+  ##
+  # Return formatted number +n+ with the appropriate units as
+  # powers of 1,000 (if +bin+ if false) or 1,024 (otherwise)
+  def num_suffix(n, bin = false)
+    p = ''
+    { T: 4, G: 3, M: 2, K: 1 }.each do |k, x|
+      v = (bin ? 1024 : 1e3)**x
+      if n > v
+        n = '%.1f' % (n / v)
+        p = k
+        break
+      end
+    end
+    "#{n}#{p}"
+  end
+
+  def like_io?(obj)
+    obj.is_a?(IO) || obj.is_a?(StringIO)
+  end
 end

data/lib/miga/daemon.rb

@@ -72,6 +72,7 @@ class MiGA::Daemon < MiGA::MiGA
     say '-----------------------------------'
     say 'MiGA:%s launched' % project.name
     say '-----------------------------------'
+    miga_say "Saving log to: #{output_file}" unless show_log?
     recalculate_status!
     load_status
     say 'Configuration options:'
@@ -111,10 +112,12 @@ class MiGA::Daemon < MiGA::MiGA
     say(*msg) if verbosity >= level
   end
 
+  alias miga_say say
+
   ##
   # Same as +l_say+ with +level = 1+
   def say(*msg)
-    super(*msg) if verbosity >= 1
+    super(logfh, *msg) if verbosity >= 1
   end
 
   ##
@@ -173,6 +176,12 @@ class MiGA::Daemon < MiGA::MiGA
       o = true if ds.ref?
       queue_job(:d, ds)
     end
+    unless show_log?
+      n = project.dataset_names.count
+      k = jobs_to_run.size + jobs_running.size
+      advance('Datasets:', n - k, n, false)
+      miga_say if k == 0
+    end
     o
   end
 
@@ -332,8 +341,9 @@ class MiGA::Daemon < MiGA::MiGA
       kill: %w[pid]
     }.each do |k, v|
       if !runopts(k).nil? && runopts(k) =~ /%(\d+\$)?[ds]/
-        runopts(k,
-                runopts(k).gsub(/%(\d+\$)?d/, '%\\1s') % v.map { |i| "{{#{i}}}" })
+        runopts(
+          k, runopts(k).gsub(/%(\d+\$)?d/, '%\\1s') % v.map { |i| "{{#{i}}}" }
+        )
       end
     end
     runopts(:format_version, 1)

data/lib/miga/daemon/base.rb

@@ -17,7 +17,7 @@ module MiGA::Daemon::Base
         if !force && v == 0 && k != :verbosity
           raise "Daemon's #{k} cannot be set to zero"
         end
-      when :shutdown_when_done
+      when :shutdown_when_done, :show_log
         v = !!v
       when :nodelist
         if v =~ /^\$/
@@ -73,4 +73,31 @@ module MiGA::Daemon::Base
   def verbosity
     runopts(:verbosity) || 1
   end
+
+  ##
+  # Writing file handler (IO) to the log file
+  def logfh
+    return $stderr if show_log?
+    return @logfh if @logfh && !@logfh.closed?
+
+    @logfh = File.open(output_file, 'w')
+  end
+
+  ##
+  # Display log instead of the progress summary
+  def show_log!
+    @show_log = true
+  end
+
+  ##
+  # Display progress summary instead of the log
+  def show_summary!
+    @runopts[:show_log] = false
+  end
+
+  ##
+  # Display log instead of the progress summary?
+  def show_log?
+    @runopts[:show_log] ||= false
+  end
 end

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 13, 1]
+  VERSION = [0.7, 16, 0]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 7, 31)
+  VERSION_DATE = Date.new(2020, 10, 13)
 
   ##
   # Reference of MiGA.

data/test/daemon_test.rb

@@ -9,7 +9,7 @@ class DaemonTest < Test::Unit::TestCase
   def setup
     initialize_miga_home(
       <<~DAEMON
-        { "maxjobs": 1, "ppn": 1, "latency": 2, "varsep": " ",
+        { "maxjobs": 1, "ppn": 1, "latency": 2, "varsep": " ", "show_log": true,
           "var": "{{key}}={{value}}", "cmd": "echo {{task_name}} >/dev/null",
           "alive": "echo 1 # {{pid}}", "type": "bash", "format_version": 1 }
       DAEMON

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.13.1
+  version: 0.7.16.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-07-31 00:00:00.000000000 Z
+date: 2020-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons