miga-base 0.7.16.12 → 0.7.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1e89d0bc90eef4ca458fbc415cbeff6eaa6c2b07ee4df77268826ecfbe0720df
-  data.tar.gz: b082d83022262e82d309e8ff70f077fe28cebddacf5b85844547ed24a8a877f7
+  metadata.gz: 78b020bda6120b5a284100b9544d7a488d385630f21b5d6d5e2bb9b365c832fb
+  data.tar.gz: d8332711fdf86308726700654ee8db91bb097f022a26e1240e62d107efa833a1
 SHA512:
-  metadata.gz: fa0746c6173b1b7a504b6d40ed0a158ef647694970f9222aeeaa5465dcad73f4647894164d0edc823b54dce837410393ac61186a6b217ff5edacbfc81a01591d
-  data.tar.gz: b8411e7c9fb0f62de20e5a909dbf7b4c998bcb18aa4fa79fbe48dc128292f8983ce1dfae54d9e3082807449b6c5cf9d4a3bb913bc9dde77c1c2d4bab6aa8fe27
+  metadata.gz: 8b766a40acf37193204e6369e07f2ecba1a67aa6141133654ab69f89b2ba9a63a959bd3f5b7fd21c56542eea7e3174cc342b05dc76b73ed27c6a2b273c546ca9
+  data.tar.gz: 6f8d791b9bf443a1eb43198625ebce48501ebaa3680be0c4cef55931b176fd2365407865e6e43cd549217ada6c67e5c86b156bd0816b80f91c388af812af810c

data/lib/miga/cli/action/doctor/base.rb

@@ -1,5 +1,5 @@
 require 'miga/cli/action'
-require 'sqlite3'
+require 'miga/sqlite'
 
 class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
 end
@@ -10,9 +10,7 @@ module MiGA::Cli::Action::Doctor::Base
   # tables saving +metric+ (:ani or :aai) and call +blk+ if the
   # file is corrupt or doesn't contain the expected structure
   def check_sqlite3_database(db_file, metric, &blk)
-    SQLite3::Database.new(db_file) do |conn|
-      conn.execute("select count(*) from #{metric}").first
-    end
+    MiGA::SQLite.new(db_file).run("select count(*) from #{metric}")
   rescue SQLite3::SQLException, SQLite3::CorruptException
     blk.call
   end
@@ -107,11 +105,7 @@ module MiGA::Cli::Action::Doctor::Base
     dist = dataset.result(:distances) or return
     path = dist.file_path(:aai_db) or return
 
-    o = []
-    SQLite3::Database.new(path) do |conn|
-      o = conn.execute('select seq2 from aai').map(&:first)
-    end
-    o
+    MiGA::SQLite.new(path).run('select seq2 from aai').map(&:first)
   end
 
   ##
@@ -120,34 +114,20 @@ module MiGA::Cli::Action::Doctor::Base
   def save_bidirectional(a, b)
     each_database_file(a) do |db_file, metric, result|
       data = nil
-      SQLite3::Database.new(db_file) do |conn|
-        data =
-          conn.execute(
-            "select seq1, seq2, #{metric}, sd, n, omega " +
-            "from #{metric} where seq2 = ? limit 1", b.name
-          ).first
-      end
+      data = MiGA::SQLite.new(db_file).run(
+        "select seq1, seq2, #{metric}, sd, n, omega " +
+        "from #{metric} where seq2 = ? limit 1", b.name
+      ).first
       next if data.nil? || data.empty?
 
       db_file_b = File.join(File.dirname(db_file), "#{b.name}.db")
       next unless File.exist?(db_file_b)
 
       data[0], data[1] = data[1], data[0]
-      SQLite3::Database.new(db_file_b) do |conn|
-        attempts = 0
-        begin
-          attempts += 1
-          conn.execute(
-            "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
-            "values(?, ?, ?, ?, ?, ?)", data
-          )
-        rescue SQLite3::BusyException => e
-          raise "Cannot populate #{db_file_b}: #{e.message}" if attempts > 3
-
-          sleep(1)
-          retry
-        end
-      end
+      MiGA::SQLite.new(db_file_b).run(
+        "insert into #{metric} (seq1, seq2, #{metric}, sd, n, omega) " +
+        "values(?, ?, ?, ?, ?, ?)", data
+      )
     end
   end
 end

data/lib/miga/daemon.rb

@@ -73,7 +73,7 @@ class MiGA::Daemon < MiGA::MiGA
     say 'MiGA:%s launched' % project.name
     say '-----------------------------------'
     miga_say "Saving log to: #{output_file}" unless show_log?
-    recalculate_status!
+    queue_maintenance
     load_status
     say 'Configuration options:'
     say @runopts.to_s
@@ -92,7 +92,7 @@ class MiGA::Daemon < MiGA::MiGA
     flush!
     if (loop_i % 12).zero?
       purge!
-      recalculate_status!
+      queue_maintenance
     end
     save_status
     sleep(latency)
@@ -100,11 +100,13 @@ class MiGA::Daemon < MiGA::MiGA
     true
   end
 
-  def recalculate_status!
-    say 'Recalculating status for all complete datasets'
-    project.each_dataset do |ds|
-      ds.recalculate_status if ds.status == :complete
-    end
+  ##
+  # Queue maintenance tasks as an analysis job
+  def queue_maintenance
+    return if bypass_maintenance?
+
+    say 'Queueing maintenance tasks'
+    queue_job(:maintenance)
   end
 
   ##
@@ -272,6 +274,11 @@ class MiGA::Daemon < MiGA::MiGA
     # Avoid single datasets hogging resources
     @jobs_to_run.rotate! rand(jobs_to_run.size)
 
+    # Prioritize project-wide jobs
+    project_jobs = @jobs_to_run.select { |i| i[:ds].nil? }
+    @jobs_to_run.delete_if { |i| i[:ds].nil? }
+    @jobs_to_run.prepend(*project_jobs)
+
     # Launch as many +jobs_to_run+ as possible
     while (hostk = next_host)
       break if jobs_to_run.empty?

data/lib/miga/daemon/base.rb

@@ -17,7 +17,7 @@ module MiGA::Daemon::Base
         if !force && v == 0 && k != :verbosity
           raise "Daemon's #{k} cannot be set to zero"
         end
-      when :shutdown_when_done, :show_log
+      when :shutdown_when_done, :show_log, :bypass_maintenance
         v = !!v
       when :nodelist
         if v =~ /^\$/
@@ -63,6 +63,12 @@ module MiGA::Daemon::Base
     !!runopts(:shutdown_when_done)
   end
 
+  ##
+  # Should the daemon ignore regular maintenance steps?
+  def bypass_maintenance?
+    !!runopts(:bypass_maintenance)
+  end
+
   ##
   # Returns the level of verbosity for the daemon as an Integer, or 1 if unset.
   # Verbosity levels are:

data/lib/miga/dataset.rb

@@ -7,7 +7,7 @@ require 'miga/metadata'
 require 'miga/dataset/result'
 require 'miga/dataset/status'
 require 'miga/dataset/hooks'
-require 'sqlite3'
+require 'miga/sqlite'
 
 ##
 # Dataset representation in MiGA
@@ -190,8 +190,7 @@ class MiGA::Dataset < MiGA::MiGA
     r = result(ref_project ? :taxonomy : :distances)
     return nil if r.nil?
 
-    db = SQLite3::Database.new(r.file_path(:aai_db))
-    db.execute(
+    MiGA::SQLite.new(r.file_path(:aai_db)).run(
       'SELECT seq2, aai FROM aai WHERE seq2 != ? ' \
       'GROUP BY seq2 ORDER BY aai DESC LIMIT ?', [name, how_many]
     )

data/lib/miga/dataset/result.rb

@@ -1,4 +1,4 @@
-require 'sqlite3'
+require 'miga/sqlite'
 require 'miga/result'
 require 'miga/dataset/base'
 require 'miga/common/with_result'
@@ -151,13 +151,13 @@ module MiGA::Dataset::Result
       db = r.file_path(db_type)
       next if db.nil? || !File.size?(db)
 
-      sqlite_db = SQLite3::Database.new db
+      sqlite_db = MiGA::SQLite.new(db)
       table = db_type[-6..-4]
-      val = sqlite_db.execute "select seq2 from #{table}"
+      val = sqlite_db.run("select seq2 from #{table}")
       next if val.empty?
 
       (val.map(&:first) - ref).each do |extra|
-        sqlite_db.execute "delete from #{table} where seq2=?", extra
+        sqlite_db.run("delete from #{table} where seq2=?", extra)
       end
     end
   end

data/lib/miga/sqlite.rb

@@ -0,0 +1,49 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'sqlite3'
+
+##
+# SQLite3 wrapper for MiGA.
+class MiGA::SQLite < MiGA::MiGA
+  class << self
+    ##
+    # Default parsing options. Supported +opts+ keys:
+    # - +:busy_attempts+: Number of times to retry when database is busy
+    #   (default: 3)
+    def default_opts(opts = {})
+      opts[:busy_attempts] ||= 3
+      opts
+    end
+  end
+
+  ##
+  # Options hash
+  attr :opts
+
+  ##
+  # Database absolute path
+  attr :path
+
+  ##
+  # Create MiGA::SQLite with database in +path+ (without opening a connection)
+  # and options +opts+ (see +.default_opts+)
+  def initialize(path, opts = {})
+    @opts = MiGA::SQLite.default_opts(opts)
+    @path = File.absolute_path(path)
+  end
+
+  ##
+  # Executes +cmd+ and returns the result
+  def run(*cmd)
+    busy_attempts ||= 0
+    conn = SQLite3::Database.new(path)
+    conn.execute(*cmd)
+  rescue SQLite3::BusyException => e
+    busy_attempts += 1
+    raise "Database busy #{path}: #{e.message}" if busy_attempts >= 3
+
+    sleep(1)
+    retry
+  end
+end

data/lib/miga/version.rb

@@ -8,7 +8,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 16, 12]
+  VERSION = [0.7, 18, 0]
 
   ##
   # Nickname for the current major.minor version.
@@ -16,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 12, 24)
+  VERSION_DATE = Date.new(2020, 12, 27)
 
   ##
   # Reference of MiGA.

data/scripts/maintenance.bash

@@ -0,0 +1,9 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+set -e
+SCRIPT="maintenance"
+# shellcheck source=scripts/miga.bash
+. "$MIGA/scripts/miga.bash" || exit 1
+
+miga doctor --only status -P "$PROJECT" -t "$CORES" -v
+

data/scripts/project_stats.bash

@@ -12,12 +12,15 @@ cd "$DIR"
 miga date > "miga-project.start"
 
 # Execute doctor
+echo "# Doctor"
 miga doctor -P "$PROJECT" -t "$CORES" -v
 
 # Index taxonomy
+echo "# Index taxonomy"
 miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active
 
 # Index metadata
+echo "# Index metadata"
 ruby -I "$MIGA/lib" \
   "$MIGA/utils/index_metadata.rb" "$PROJECT" "miga-project.metadata.db"
 

data/test/daemon_test.rb

@@ -93,7 +93,7 @@ class DaemonTest < Test::Unit::TestCase
       0 => /-{20}\n/,
       1 => /MiGA:#{p.name} launched/,
       2 => /-{20}\n/,
-      6 => /Probing running jobs\n/
+      8 => /Probing running jobs\n/
     }.each { |k, v| assert_match(v, l[k], "unexpected line: #{k}") }
   ensure
     begin
@@ -160,8 +160,9 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal(0, d1.jobs_running.size)
     assert_equal(0, d1.jobs_to_run.size)
     capture_stderr { d1.in_loop }
+    # 3 dataset jobs + 1 maintenance job:
     assert_equal(1, d1.jobs_running.size)
-    assert_equal(2, d1.jobs_to_run.size)
+    assert_equal(3, d1.jobs_to_run.size)
   end
 
   def test_maxjobs_runopts
@@ -172,7 +173,7 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal(0, d1.jobs_to_run.size)
     capture_stderr { d1.in_loop }
     assert_equal(2, d1.jobs_running.size)
-    assert_equal(1, d1.jobs_to_run.size)
+    assert_equal(2, d1.jobs_to_run.size)
   end
 
   def test_load_status
@@ -232,6 +233,7 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   def test_shutdown_when_done
+    daemon.runopts(:bypass_maintenance, true)
     daemon.runopts(:shutdown_when_done, true)
     out = capture_stderr { assert { !daemon.in_loop } }.string
     assert_match(/Nothing else to do/, out)
@@ -317,4 +319,20 @@ class DaemonTest < Test::Unit::TestCase
     out = capture_stderr { d1.in_loop }.string
     assert_match(/Daemon loop start/, out)
   end
+
+  def test_bypass_maintenance
+    # Default (run maintenance)
+    d = daemon(0)
+    d.runopts(:latency, 0, true)
+    capture_stderr { d.in_loop }
+    assert_equal(1, d.jobs_running.size)
+    assert_equal(:maintenance, d.jobs_running.first[:job])
+
+    # Bypassing maintenance
+    d = daemon(1)
+    d.runopts(:latency, 0, true)
+    d.runopts(:bypass_maintenance, true)
+    capture_stderr { d.in_loop }
+    assert_equal([], d.jobs_running)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.7.16.12
+  version: 0.7.18.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-12-24 00:00:00.000000000 Z
+date: 2020-12-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -196,6 +196,7 @@ files:
 - lib/miga/result/dates.rb
 - lib/miga/result/source.rb
 - lib/miga/result/stats.rb
+- lib/miga/sqlite.rb
 - lib/miga/tax_dist.rb
 - lib/miga/tax_index.rb
 - lib/miga/taxonomy.rb
@@ -211,6 +212,7 @@ files:
 - scripts/essential_genes.bash
 - scripts/haai_distances.bash
 - scripts/init.bash
+- scripts/maintenance.bash
 - scripts/miga.bash
 - scripts/mytaxa.bash
 - scripts/mytaxa_scan.bash