miga-base 0.7.15.0 → 0.7.16.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0160b454e26886637f28dea78ab57e66b1c53ee3fca610c64339f3265ce86afb
-  data.tar.gz: 6df1a7bfc7ebf0493265cd6be8f8600a575b3c20f07ec43f7c25b8de3035b5c8
+  metadata.gz: 51173aea1659d14d35f485b22b4371bdf8cf9d16ae81c673d09c25cf3e36d1ac
+  data.tar.gz: 4c6cbaed4bc9f0ea2f71d75ccfd22e3b0ac38e62c138166b768feb91c9a804b3
 SHA512:
-  metadata.gz: 00245e9a4d698ed335f1777bd1f9156ecc2ba325c4ef66f42f01f9e59c70e4bfb42a588a7be62271dd55df73e2fc737a29e24253487ba55e9d8645a43508d91c
-  data.tar.gz: 12c8beb33f81fc7114d957a59982d4bfab1deceb21db27dde1a93f474d4ab5918308ac04d351c86b9a71f8a6804e0311b3230ccbad16de320523246f1e7e3f9b
+  metadata.gz: 6e00443678373cb3125e1d73d111ff909cfaa092c92ffd00089e2d122bad0f298133e4c4b2da00370d4ff56e3d1281b8e3ebe316e23fa675b73b217b6045e4e8
+  data.tar.gz: f5d5f2981f86e0a79a1c96b81cdb76853eeacd11a14c23d630bcf067d390074d40273159b2b95c3ea33d88d1ca1c66bce19338809af410e48a0db2138d4974d3

data/README.md

@@ -11,11 +11,12 @@
 
 For additional information on MiGA, visit:
 
-* [MiGA Online][miga-online]: The Microbial Genomes Atlas Online.
+* [MiGA Online][miga-online]: The Microbial Genomes Atlas Online
+* [MiGA@XSEDE][miga-at-xsede]: The MiGA@XSEDE Gateway
 * [MiGA users list][mailing-list]:
-  Forum to discuss with other users and developers.
-* [MiGA manual][manual]: The definitive guide to MiGA.
-* [MiGA API docs][rubydoc]: Inner-workings of the `miga-base` gem.
+  Forum to discuss with other users and developers
+* [MiGA manual][manual]: The definitive guide to MiGA
+* [MiGA API docs][rubydoc]: Inner-workings of the `miga-base` gem
 * [MiGA Web][miga-web]: MiGA on Rails!
 
 # For the impatient
@@ -39,7 +40,8 @@ You have two options:
 
 Developed and maintained by [Luis M. Rodriguez-R][lrr]. MiGA is the result of a
 collaboration between [Kostas Lab][kostas] at the Georgia Institute of
-Technology and [RDP][rdp] at Michigan State University.
+Technology, [Rodriguez-R Lab][lrr] at the University of Innsbruck,
+and [RDP][rdp] at Michigan State University.
 
 See also the [complete list of contributors](manual/part1/contributors.md).
 
@@ -47,7 +49,7 @@ See also the [complete list of contributors](manual/part1/contributors.md).
 
 See [LICENSE](LICENSE).
 
-[lrr]: http://lmrodriguezr.github.io/
+[lrr]: https://rodriguez-r.com/
 [mailing-list]: http://support.microbial-genomes.org/
 [manual]: https://manual.microbial-genomes.org/
 [rubydoc]: http://www.rubydoc.info/github/bio-miga/miga
@@ -55,5 +57,6 @@ See [LICENSE](LICENSE).
 [miga-web]: https://github.com/bio-miga/miga-web
 [miga-gui]: https://github.com/bio-miga/miga-gui
 [miga-online]: http://microbial-genomes.org/
+[miga-at-xsede]: https://xsede.microbial-genomes.org/
 [kostas]: http://enve-omics.gatech.edu/
 [rdp]: http://rdp.cme.msu.edu/

data/lib/miga/cli/action/doctor.rb

@@ -7,6 +7,7 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   include MiGA::Cli::Action::Doctor::Base
 
   def parse_cli
+    cli.defaults = { threads: 1 }
     cli.defaults = Hash[@@OPERATIONS.keys.map { |i| [i, true] }]
     cli.parse do |opt|
       operation_n = Hash[@@OPERATIONS.map { |k, v| [v[0], k] }]
@@ -24,6 +25,10 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
         @@OPERATIONS.each_key { |i| cli[i] = false }
         cli[op_k] = true
       end
+      opt.on(
+        '-t', '--threads INT', Integer,
+        "Concurrent threads to use. By default: #{cli[:threads]}"
+      ) { |v| cli[:threads] = v }
     end
   end
 
@@ -37,6 +42,7 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   @@OPERATIONS = {
     status: ['status', 'Update metadata status of all datasets'],
     db: ['databases', 'Check integrity of database files'],
+    bidir: ['bidirectional', 'Check distances are bidirectional'],
     dist: ['distances', 'Check distance summary tables'],
     files: ['files', 'Check for outdated files'],
     cds: ['cds', 'Check for gzipped genes and proteins'],
@@ -58,11 +64,19 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   # Perform status operation with MiGA::Cli +cli+
   def check_status(cli)
     cli.say 'Updating metadata status'
-    n, k = cli.load_project.dataset_names.size, 0
-    cli.load_project.each_dataset do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      d.recalculate_status
+    p = cli.load_project
+    n = p.dataset_names.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        cli.load_project.each_dataset do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          d.recalculate_status if k % cli[:threads] == i
+        end
+      end
     end
+    Process.waitall
     cli.say
   end
 
@@ -70,18 +84,59 @@ class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
   # Perform databases operation with MiGA::Cli +cli+
   def check_db(cli)
     cli.say 'Checking integrity of databases'
-    n, k = cli.load_project.dataset_names.size, 0
-    cli.load_project.each_dataset do |d|
-      cli.advance('Datasets:', k += 1, n, false)
-      each_database_file(d) do |db_file, metric, result|
-        check_sqlite3_database(db_file, metric) do
-          cli.say("  > Removing malformed database from #{d.name}:#{result}   ")
-          File.unlink(db_file)
-          r = d.result(result) or next
-          [r.path(:done), r.path].each { |f| File.unlink(f) if File.exist?(f) }
+    p = cli.load_project
+    n = p.dataset_names.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        p.each_dataset do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          next unless k % cli[:threads] == i
+          each_database_file(d) do |db_file, metric, result|
+            check_sqlite3_database(db_file, metric) do
+              cli.say(
+                "  > Removing malformed database from #{d.name}:#{result}   "
+              )
+              File.unlink(db_file)
+              r = d.result(result) or next
+              [r.path(:done), r.path].each do |f|
+                File.unlink(f) if File.exist?(f)
+              end
+            end
+          end
+        end
+      end
+    end
+    Process.waitall
+    cli.say
+  end
+
+  ##
+  # Perform bidirectional operation with MiGA::Cli +cli+
+  def check_bidir(cli)
+    cli.say 'Checking if reference distances are bidirectional'
+    ref_ds = cli.load_project.each_dataset.select(&:ref?)
+    ref_names = ref_ds.map(&:name)
+    n = ref_ds.size
+    (0 .. cli[:threads] - 1).map do |i|
+      Process.fork do
+        k = 0
+        ref_ds.each do |d|
+          k += 1
+          cli.advance('Datasets:', k, n, false) if i == 0
+          next unless k % cli[:threads] == i
+
+          saved = saved_targets(d)
+          next if saved.nil?
+
+          (ref_names - saved).each do |k|
+            save_bidirectional(cli.load_project.dataset(k), d)
+          end
         end
       end
     end
+    Process.waitall
     cli.say
   end
 

data/lib/miga/cli/action/doctor/base.rb

@@ -99,4 +99,55 @@ module MiGA::Cli::Action::Doctor::Base
     cli.say '- Removing tables, recompute'
     res.remove!
   end
+
+  ##
+  # Returns all targets identified by AAI
+  def saved_targets(dataset)
+    # Return nil if distance or database are not retrievable
+    dist = dataset.result(:distances) or return
+    path = dist.file_path(:aai_db) or return
+
+    o = []
+    SQLite3::Database.new(path) do |conn|
+      o = conn.execute('select seq2 from aai').map(&:first)
+    end
+    o
+  end
+
+  ##
+  # Saves all the distance estimates in +a+ -> +b+ into the +b+ databases
+  # (as +b+ -> +a+), where both +a+ and +b+ are MiGA::Dataset objects
+  def save_bidirectional(a, b)
+    each_database_file(a) do |db_file, metric, result|
+      data = nil
+      SQLite3::Database.new(db_file) do |conn|
+        data =
+          conn.execute(
+            "select seq1, seq2, #{metric}, sd, n, omega " +
+            "from #{metric} where seq2 = ? limit 1", b.name
+          ).first
+      end
+      next if data.nil? || data.empty?
+
+      db_file_b = File.join(File.dirname(db_file), "#{b.name}.db")
+      next unless File.exist?(db_file_b)
+
+      data[0], data[1] = data[1], data[0]
+      SQLite3::Database.new(db_file_b) do |conn|
+        attempts = 0
+        begin
+          attempts += 1
+          conn.execute(
+            "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
+            "values(?, ?, ?, ?, ?, ?)", data
+          )
+        rescue SQLite3::BusyException => e
+          raise "Cannot populate #{db_file_b}: #{e.message}" if attempts > 3
+
+          sleep(1)
+          retry
+        end
+      end
+    end
+  end
 end

data/lib/miga/cli/action/get.rb

@@ -70,7 +70,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
     glob = get_sub_cli
     p = cli.load_project
     glob.each do |sub_cli|
-      rd = create_remote_dataset(sub_cli)
+      rd = create_remote_dataset(sub_cli, p)
       next if rd.nil?
 
       if sub_cli[:get_md]
@@ -115,7 +115,7 @@ class MiGA::Cli::Action::Get < MiGA::Cli::Action
     glob
   end
 
-  def create_remote_dataset(sub_cli)
+  def create_remote_dataset(sub_cli, p)
     sub_cli.ensure_par(dataset: '-D', ids: '-I')
     unless sub_cli[:api_key].nil?
       ENV["#{sub_cli[:universe].to_s.upcase}_API_KEY"] = sub_cli[:api_key]

data/lib/miga/common.rb

@@ -52,10 +52,39 @@ class MiGA::MiGA
   # 1,000 otherwise.
   # The report goes to $stderr iff --verborse
   def advance(step, n = 0, total = nil, bin = true)
-    adv = total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
-      ('%.1f%% (%s/%s)' % [100.0 * n / total,
-                           num_suffix(n, bin), num_suffix(total, bin)])
-    $stderr.print("[%s] %s %s    \r" % [Time.now, step, adv])
+    # Initialize advance timing
+    @_advance_time ||= { last: nil, n: 0, avg: nil }
+    if n <= 1 || @_advance_time[:n] > n
+      @_advance_time[:last] = nil
+      @_advance_time[:n] = 0
+      @_advance_time[:avg]  = nil
+    end
+
+    # Estimate timing
+    unless total.nil? || @_advance_time[:last].nil? || @_advance_time[:n] == n
+      this_time = Time.now - @_advance_time[:last]
+      @_advance_time[:avg] ||= this_time / (n - @_advance_time[:n])
+      @_advance_time[:avg] = 0.99 * @_advance_time[:avg] + 0.01 * this_time
+    end
+    @_advance_time[:last] = Time.now
+    @_advance_time[:n] = n
+
+    # Report
+    adv_vals = [100.0 * n / total, num_suffix(n, bin), num_suffix(total, bin)]
+    adv =
+      total.nil? ? (n == 0 ? '' : num_suffix(n, bin)) :
+        ('%.1f%% (%s/%s)' % adv_vals)
+    left =
+      if @_advance_time[:avg].nil?
+        ''
+      else
+        left_time = @_advance_time[:avg] * (total - n) / 60
+        left_time < 0.01 ? '         ' :
+          left_time < 1 ? ('%.0fs left' % (left_time * 60)) :
+          left_time > 60 ? ('%.1fh left' % (left_time / 60)) :
+          ('%.1fm left' % left_time)
+      end
+    $stderr.print("[%s] %s %s %s    \r" % [Time.now, step, adv, left])
   end
 
   ##

data/lib/miga/daemon/base.rb

@@ -77,7 +77,11 @@ module MiGA::Daemon::Base
   ##
   # Writing file handler (IO) to the log file
   def logfh
-    show_log? ? $stderr : (@logfh ||= File.open(output_file, 'w'))
+    @logfh ||= nil
+    return $stderr if show_log?
+    return @logfh if @logfh && !@logfh.closed?
+
+    @logfh = File.open(output_file, 'w')
   end
 
   ##

data/lib/miga/json.rb

@@ -34,17 +34,29 @@ class MiGA::Json < MiGA::MiGA
     # +opts+.
     def parse(path, opts = {})
       opts = default_opts(opts)
-      cont = opts[:contents] ? path : File.read(path)
-      raise "Empty descriptor: #{opts[:contents] ? "''" : path}." if cont.empty?
 
-      y = JSON.parse(cont,
-                     symbolize_names: opts[:symbolize],
-                     create_additions: opts[:additions])
+      # Read JSON
+      cont = path
+      12.times do
+        cont = File.read(path)
+        break unless cont.empty?
+        sleep 1 # Wait up to 12 seconds for racing processes (iff empty file)
+      end unless opts[:contents]
+      raise "Empty descriptor: #{opts[:contents] ? "''" : path}" if cont.empty?
+
+      # Parse JSON
+      params = { symbolize_names: opts[:symbolize],
+                 create_additions: opts[:additions] }
+      y = JSON.parse(cont, params)
+
+      # Add defaults
       unless opts[:default].nil?
         opts[:default] = parse(opts[:default]) if opts[:default].is_a? String
         y.each { |k, v| opts[:default][k] = v }
         y = opts[:default]
       end
+
+      # Return
       y
     end
 

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 15, 0]
+  VERSION = [0.7, 16, 3]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 8, 12)
+  VERSION_DATE = Date.new(2020, 10, 27)
 
   ##
   # Reference of MiGA.

data/scripts/project_stats.bash

@@ -12,7 +12,7 @@ cd "$DIR"
 miga date > "miga-project.start"
 
 # Execute doctor
-miga doctor -P "$PROJECT" -v
+miga doctor -P "$PROJECT" -t "$CORES" -v
 
 # Index taxonomy
 miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.15.0
+  version: 0.7.16.3
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-08-12 00:00:00.000000000 Z
+date: 2020-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons