miga-base 0.7.3.0 → 0.7.7.0

data/lib/miga/taxonomy/base.rb

@@ -2,49 +2,54 @@
 # @license Artistic-2.0
 
 class MiGA::Taxonomy < MiGA::MiGA
-
   class << self
-
     ##
-    # Returns cannonical rank (Symbol) for the +rank+ String.
+    # Returns cannonical rank (Symbol) for the +rank+ String
     def normalize_rank(rank)
       return rank.to_sym if @@_KNOWN_RANKS_H[rank.to_sym]
+
       rank = rank.to_s.downcase
       return nil if rank == 'no rank'
+
       rank = @@RANK_SYNONYMS[rank] unless @@RANK_SYNONYMS[rank].nil?
       rank = rank.to_sym
       return nil unless @@_KNOWN_RANKS_H[rank]
+
       rank
     end
 
     ##
-    # Initialize from JSON-derived Hash +o+.
+    # Initialize from JSON-derived Hash +o+
     def json_create(o)
       new(o['str'], nil, o['alt'])
     end
 
-    def KNOWN_RANKS() @@KNOWN_RANKS; end
-    def LONG_RANKS() @@LONG_RANKS; end
+    def KNOWN_RANKS()
+      @@KNOWN_RANKS
+    end
 
+    def LONG_RANKS()
+      @@LONG_RANKS
+    end
   end
-
 end
 
 module MiGA::Taxonomy::Base
-
   ##
-  # Cannonical ranks.
+  # Cannonical ranks
   @@KNOWN_RANKS = %w{ns d k p c o f g s ssp str ds}.map { |r| r.to_sym }
-  @@_KNOWN_RANKS_H = Hash[ @@KNOWN_RANKS.map { |i| [i, true] } ]
+  @@_KNOWN_RANKS_H = Hash[@@KNOWN_RANKS.map { |i| [i, true] }]
 
   ##
-  # Long names of the cannonical ranks.
-  @@LONG_RANKS = { root: 'root', ns: 'namespace', d: 'domain', k: 'kingdom',
+  # Long names of the cannonical ranks
+  @@LONG_RANKS = {
+    root: 'root', ns: 'namespace', d: 'domain', k: 'kingdom',
     p: 'phylum', c: 'class', o: 'order', f: 'family', g: 'genus', s: 'species',
-    ssp: 'subspecies', str: 'strain', ds: 'dataset' }
+    ssp: 'subspecies', str: 'strain', ds: 'dataset'
+  }
 
   ##
-  # Synonms for cannonical ranks.
+  # Synonms for cannonical ranks
   @@RANK_SYNONYMS = {
     'namespace' => 'ns',
     'domain' => 'd', 'superkingdom' => 'd',
@@ -59,5 +64,4 @@ module MiGA::Taxonomy::Base
     'strain' => 'str', 'isolate' => 'str', 'culture' => 'str',
     'dataset' => 'ds', 'organism' => 'ds', 'genome' => 'ds', 'specimen' => 'ds'
   }
-
 end

data/lib/miga/version.rb

@@ -1,16 +1,14 @@
-
 require 'date'
 
 ##
 # High-level minimal requirements for the MiGA::MiGA class.
 module MiGA
-
   ##
   # Current version of MiGA. An Array with three values:
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.7, 3, 0]
+  VERSION = [0.7, 7, 0]
 
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +16,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 4, 22)
+  VERSION_DATE = Date.new(2020, 6, 4)
 
   ##
   # Reference of MiGA.
@@ -26,33 +24,38 @@ module MiGA
     'The Microbial Genomes Atlas (MiGA) webserver: taxonomic and gene ' \
     'diversity analysis of Archaea and Bacteria at the whole genome level. ' \
     'Nucleic Acids Research 46(W1):W282-W288. doi:10.1093/nar/gky467.'
-
 end
 
 class MiGA::MiGA
-
   include MiGA
 
   ##
-  # Major.minor version as Float.
-  def self.VERSION ; VERSION[0] ; end
+  # Major.minor version as Float
+  def self.VERSION
+    VERSION[0]
+  end
 
   ##
-  # Complete version as string.
-  def self.FULL_VERSION ; VERSION.join('.') ; end
+  # Complete version as string
+  def self.FULL_VERSION
+    VERSION.join('.')
+  end
 
   ##
-  # Complete version with nickname and date as string.
+  # Complete version with nickname and date as string
   def self.LONG_VERSION
     "MiGA #{VERSION.join('.')} - #{VERSION_NAME} - #{VERSION_DATE}"
   end
 
   ##
-  # Date of the current gem release.
-  def self.VERSION_DATE ; VERSION_DATE ; end
+  # Date of the current gem release
+  def self.VERSION_DATE
+    VERSION_DATE
+  end
 
   ##
-  # Reference of MiGA.
-  def self.CITATION ; CITATION ; end
-
+  # Reference of MiGA
+  def self.CITATION
+    CITATION
+  end
 end

data/scripts/project_stats.bash

@@ -11,6 +11,9 @@ cd "$DIR"
 # Initialize
 miga date > "miga-project.start"
 
+# Execute doctor
+miga doctor -P "$PROJECT" -v
+
 # Index taxonomy
 miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active
 

data/scripts/stats.bash

@@ -14,7 +14,7 @@ miga date > "$DATASET.start"
 # Calculate statistics
 for i in raw_reads trimmed_fasta assembly cds essential_genes ssu distances taxonomy ; do
   echo "# $i"
-  miga stats --compute-and-save -P "$PROJECT" -D "$DATASET" -r $i
+  miga stats --compute-and-save --ignore-empty -P "$PROJECT" -D "$DATASET" -r $i
 done
 
 # Finalize

data/test/common_test.rb

@@ -1,10 +1,7 @@
 require 'test_helper'
 
 class CommonTest < Test::Unit::TestCase
-
-  def setup
-    #$jruby_tests = !ENV["JRUBY_TESTS"].nil?
-  end
+  include TestHelper
 
   def test_debug
     assert_respond_to(MiGA::MiGA, :DEBUG)
@@ -28,7 +25,6 @@ class CommonTest < Test::Unit::TestCase
     assert_respond_to(MiGA::MiGA, :DEBUG)
     assert_respond_to(MiGA::MiGA, :DEBUG_ON)
     assert_respond_to(MiGA::MiGA, :DEBUG_OFF)
-    #omit_if($jruby_tests, "JRuby doesn't like interceptions.")
     MiGA::MiGA.DEBUG_TRACE_ON
     err = capture_stderr do
       MiGA::MiGA.DEBUG 'Dandadi'
@@ -45,9 +41,8 @@ class CommonTest < Test::Unit::TestCase
   end
 
   def test_generic_transfer
-    $tmp = Dir.mktmpdir
-    hello = File.expand_path('Hello', $tmp)
-    world = File.expand_path('World', $tmp)
+    hello = File.expand_path('Hello', tmpdir)
+    world = File.expand_path('World', tmpdir)
     assert_respond_to(File, :generic_transfer)
     FileUtils.touch(hello)
     File.generic_transfer(hello, world, :symlink)
@@ -67,8 +62,6 @@ class CommonTest < Test::Unit::TestCase
       File.generic_transfer(hello, world, :monkey)
     end
     assert_path_not_exist(world, 'A monkey shouldn\'t create files.')
-  ensure
-    FileUtils.rm_rf $tmp
   end
 
   def test_miga_name
@@ -79,5 +72,4 @@ class CommonTest < Test::Unit::TestCase
     assert_not_predicate('C3-PO', :miga_name?)
     assert_equal("123\n1\n", '1231'.wrap_width(3))
   end
-
 end

data/test/daemon_helper.rb

@@ -0,0 +1,38 @@
+##
+# Helper functions to test daemons.
+# The class MUST also include +TestHelper+
+module DaemonHelper
+  def daemon(i = 0)
+    @daemon ||= {}
+    @daemon[project(i).name] ||= MiGA::Daemon.new(project(i))
+  end
+
+  def helper_datasets_with_results(n = 1, project_i = 0)
+    p1 = project(project_i)
+    Array.new(n) do |i|
+      d = "d#{i}"
+      FileUtils.touch(
+        File.join(p1.path, 'data', '02.trimmed_reads', "#{d}.1.clipped.fastq")
+      )
+      FileUtils.touch(
+        File.join(p1.path, 'data', '02.trimmed_reads', "#{d}.done")
+      )
+      p1.add_dataset(MiGA::Dataset.new(p1, d, true).name).tap do |ds|
+        ds.first_preprocessing(true)
+      end
+    end
+  end
+
+  def helper_daemon_launch_job(project_i = 0)
+    declare_forks
+    d1 = daemon(project_i)
+    helper_datasets_with_results(1, project_i).first.inactivate!
+    assert_equal(0, d1.jobs_to_run.size, 'The queue should be empty')
+    capture_stderr { d1.check_project }
+    assert_equal(1, d1.jobs_to_run.size, 'The queue should have one job')
+    capture_stderr { d1.flush! }
+    sleep(1)
+    assert_equal(0, d1.jobs_to_run.size, 'There should be nothing running')
+    assert_equal(1, d1.jobs_running.size, 'There should be one job running')
+  end
+end

data/test/daemon_test.rb

@@ -1,69 +1,46 @@
 require 'test_helper'
+require 'daemon_helper'
 require 'miga/daemon'
 
 class DaemonTest < Test::Unit::TestCase
+  include TestHelper
+  include DaemonHelper
 
   def setup
-    $jruby_tests = !ENV['JRUBY_TESTS'].nil?
-    $tmp = Dir.mktmpdir
-    ENV['MIGA_HOME'] = $tmp
-    FileUtils.touch(File.expand_path('.miga_rc', ENV['MIGA_HOME']))
-    daemon_json = File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
-    File.open(daemon_json, 'w') do |fh|
-      fh.puts '{"maxjobs":1,"ppn":1,"latency":2,"varsep":" ",
-        "var":"{{key}}={{value}}","cmd":"echo {{task_name}} >/dev/null",
-        "alive":"echo 1 # {{pid}}","type":"bash","format_version":1}'
-    end
-    $p1 = MiGA::Project.new(File.expand_path('project1', $tmp))
-    $d1 = MiGA::Daemon.new($p1)
-  end
-
-  def teardown
-    FileUtils.rm_rf $tmp
-    ENV['MIGA_HOME'] = nil
-  end
-
-  def helper_datasets_with_results(n = 1)
-    p1 = $p1
-    Array.new(n) do |i|
-      d = "d#{i}"
-      FileUtils.touch(File.expand_path(
-        "data/02.trimmed_reads/#{d}.1.clipped.fastq", p1.path
-      ))
-      FileUtils.touch(File.expand_path(
-        "data/02.trimmed_reads/#{d}.done", p1.path
-      ))
-      p1.add_dataset(MiGA::Dataset.new(p1, d, true).name).tap do |ds|
-        ds.first_preprocessing(true)
-      end
-    end
+    initialize_miga_home(
+      <<~DAEMON
+        { "maxjobs": 1, "ppn": 1, "latency": 2, "varsep": " ",
+          "var": "{{key}}={{value}}", "cmd": "echo {{task_name}} >/dev/null",
+          "alive": "echo 1 # {{pid}}", "type": "bash", "format_version": 1 }
+      DAEMON
+    )
   end
 
   def test_check_project
-    d1 = $d1
-    helper_datasets_with_results.first.inactivate!
-    out = capture_stderr { d1.check_project }
-    assert_match(/Queueing miga-project:p/, out.string)
+    d1 = daemon(1)
+    helper_datasets_with_results(1, 1).first.inactivate!
+    out = capture_stderr { d1.check_project }.string
+    assert_match(/Queueing miga-project:p/, out)
     assert_equal(1, d1.jobs_to_run.size)
     assert_equal(:p, d1.jobs_to_run.first[:job])
     assert_equal('project1:p:miga-project', d1.get_job(:p)[:task_name])
   end
 
   def test_check_datasets
-    p = $p1
-    d = $d1
+    p = project
+    d = daemon
     d.runopts(:maxjobs, 0, true)
     assert_empty(d.jobs_to_run)
     ds = p.add_dataset('ds1')
     d.check_datasets
     assert_empty(d.jobs_to_run)
     FileUtils.cp(
-      File.expand_path('daemon/daemon.json', p.path),
-      File.expand_path('data/01.raw_reads/ds1.1.fastq', p.path)
+      File.join(p.path, 'daemon/daemon.json'),
+      File.join(p.path, 'data/01.raw_reads/ds1.1.fastq')
     )
     FileUtils.cp(
-      File.expand_path('daemon/daemon.json', p.path),
-      File.expand_path('data/01.raw_reads/ds1.done', p.path)
+      File.join(p.path, 'daemon/daemon.json'),
+      File.join(p.path, 'data/01.raw_reads/ds1.done')
     )
     ds.first_preprocessing(true)
     out = capture_stderr do
@@ -71,13 +48,13 @@ class DaemonTest < Test::Unit::TestCase
     end
     assert_match(/Queueing #{ds.name}:d/, out.string)
     assert_equal(1, d.jobs_to_run.size)
-    assert_equal('echo project1:d:ds1 >/dev/null', d.jobs_to_run.first[:cmd])
+    assert_equal('echo project0:d:ds1 >/dev/null', d.jobs_to_run.first[:cmd])
     assert_equal(d.jobs_to_run.first, d.get_job(:d, ds))
   end
 
   def test_in_loop
-    p = $p1
-    d = $d1
+    p = project
+    d = daemon
     d.runopts(:latency, 0, true)
     assert_nil(d.loop_i)
     assert_nil(d.last_alive)
@@ -88,20 +65,21 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal(11, d.loop_i)
     out = capture_stderr { d.in_loop }.string
     assert_match(/Probing running jobs/, out)
-    assert_equal(0, d.loop_i)
+    assert_equal(12, d.loop_i)
   end
 
   def test_start
-    p = $p1
-    d = $d1
+    p = project
+    d = daemon
     d.runopts(:latency, 0, true)
     assert_equal(0, d.latency)
-    omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
-    child = $child = d.daemon(:start, ['--shush'])
+
+    declare_forks
+    child = d.daemon(:start, ['--shush'])
     assert_not_nil(child)
     assert_gt(child, 1)
     assert_equal(0, `ps -p "#{child}" -o ppid=`.strip.to_i,
-      'The daemon process should be detached')
+                 'The daemon process should be detached')
     sleep(3)
     assert_path_exist(d.pid_file)
     out = capture_stderr { d.stop }.string
@@ -109,6 +87,7 @@ class DaemonTest < Test::Unit::TestCase
     assert_match(/Sending termination message/, out)
     assert_path_not_exist(d.pid_file)
     assert_path_exist(d.output_file)
+    assert_equal(1, d.verbosity)
     l = File.readlines(d.output_file)
     {
       0 => /-{20}\n/,
@@ -118,23 +97,24 @@ class DaemonTest < Test::Unit::TestCase
     }.each { |k, v| assert_match(v, l[k], "unexpected line: #{k}") }
   ensure
     begin
-      Process.kill('KILL', $child) if !$child.nil?
+      Process.kill('KILL', child) unless child.nil?
     rescue Errno::ESRCH
       false
     end
   end
 
   def test_last_alive
-    p = MiGA::Project.new(File.expand_path('last_alive', $tmp))
+    p = MiGA::Project.new(tmpfile('last_alive'))
     d = MiGA::Daemon.new(p)
     assert_nil(d.last_alive)
-    omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
+
+    declare_forks
     d.declare_alive
     assert_lt(d.last_alive, Time.now)
   end
 
   def test_options
-    d1 = $d1
+    d1 = daemon
     assert_respond_to(d1, :default_options)
     assert_equal(:normal, d1.default_options[:dir_mode])
     assert_equal(2, d1.runopts(:latency))
@@ -150,17 +130,17 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   def test_say
-    out = capture_stderr { $d1.say 'Olm' }.string
+    out = capture_stderr { daemon.say 'Olm' }.string
     assert_match(/^\[.*\] Olm/, out)
   end
 
   def test_terminate
-    d = $d1
+    d = daemon
     assert_not_predicate(d, :active?)
     assert_path_not_exist(d.alive_file)
     assert_path_not_exist(d.terminated_file)
 
-    omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
+    declare_forks
     d.declare_alive
     assert_predicate(d, :active?)
     assert_not_nil(d.last_alive)
@@ -175,7 +155,7 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   def test_maxjobs_json
-    d1 = $d1
+    d1 = daemon
     helper_datasets_with_results(3)
     assert_equal(0, d1.jobs_running.size)
     assert_equal(0, d1.jobs_to_run.size)
@@ -185,7 +165,7 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   def test_maxjobs_runopts
-    d1 = $d1
+    d1 = daemon
     helper_datasets_with_results(3)
     d1.runopts(:maxjobs, 2)
     assert_equal(0, d1.jobs_running.size)
@@ -196,8 +176,8 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   def test_load_status
-    d1 = $d1
-    p1 = $p1
+    d1 = daemon
+    p1 = project
     assert_equal(0, d1.jobs_running.size)
     assert_nil(d1.load_status)
     assert_equal(0, d1.jobs_running.size)
@@ -214,8 +194,8 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   def test_flush
-    d1 = $d1
-    p1 = $p1
+    d1 = daemon
+    p1 = project
     helper_datasets_with_results
     p1.add_dataset(MiGA::Dataset.new(p1, 'd1').name)
     MiGA::Project.RESULT_DIRS.keys.each { |i| p1.metadata["run_#{i}"] = false }
@@ -234,8 +214,8 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   def test_next_host
-    d1 = $d1
-    f = File.join($tmp, 'nodes.txt')
+    d1 = daemon
+    f = tmpfile('nodes.txt')
     File.open(f, 'w') { |h| h.puts 'localhost' }
     assert_equal(true, d1.next_host)
     out = capture_stderr { d1.runopts(:nodelist, f) }.string
@@ -243,7 +223,7 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal(true, d1.next_host)
     d1.runopts(:type, 'ssh')
     assert_equal(0, d1.next_host)
-    f = File.join($p1.path, 'daemon', 'status.json')
+    f = File.join(project.path, 'daemon', 'status.json')
     File.open(f, 'w') do |h|
       h.puts '{"jobs_running":[{"job":"p","hostk":0}], "jobs_to_run":[]}'
     end
@@ -252,77 +232,89 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   def test_shutdown_when_done
-    $d1.runopts(:shutdown_when_done, true)
-    out = capture_stderr { assert { !$d1.in_loop } }.string
+    daemon.runopts(:shutdown_when_done, true)
+    out = capture_stderr { assert { !daemon.in_loop } }.string
     assert_match(/Nothing else to do/, out)
   end
 
   def test_update_format_0
-    f = File.join($tmp, 'daemon.json')
+    f = tmpfile('daemon.json')
     File.open(f, 'w') do |fh|
-      fh.puts '{"maxjobs":1,"ppn":1,"latency":2,"varsep":" ",
-        "var":"%1$s=%1$s","cmd":"echo %1$s","alive":"echo %1$d","type":"bash"}'
+      fh.puts(
+        <<~DAEMON
+          { "maxjobs": 1, "ppn": 1, "latency": 2, "varsep": " ",
+            "var": "%1$s=%1$s", "cmd": "echo %1$s", "alive": "echo %1$d",
+            "type": "bash" }
+        DAEMON
+      )
     end
-    d2 = MiGA::Daemon.new($p1, f)
+    d2 = MiGA::Daemon.new(project, f)
     assert_equal('echo {{script}}', d2.runopts(:cmd))
     assert_equal('echo {{pid}}', d2.runopts(:alive))
   end
 
   def test_launch_job_bash
-    t = File.join($tmp, 'launch_job_bash')
-    $d1.runopts(:type, 'bash')
-    $d1.runopts(:cmd, "echo {{task_name}} > '#{t}'")
+    t = tmpfile('launch_job_bash')
+    daemon.runopts(:type, 'bash')
+    daemon.runopts(:cmd, "echo {{task_name}} > '#{t}'")
     helper_daemon_launch_job
-    assert_equal("project1:p:miga-project\n", File.read(t))
+    assert_equal("project0:p:miga-project\n", File.read(t))
   end
 
   def test_launch_job_ssh
-    d1 = $d1
-    t = File.join($tmp, 'launch_job_ssh')
+    d1 = daemon(1)
+    t = tmpfile('launch_job_ssh')
     d1.runopts(:type, 'ssh')
     d1.runopts(:cmd, "echo {{task_name}} > '#{t}'")
-    f = File.join($tmp, 'nodes.txt')
+    f = tmpfile('nodes.txt')
     File.open(f, 'w') { |h| h.puts 'localhost' }
     assert_raise('Unset environment variable: $MIGA_TEST_NODELIST') do
       d1.runopts(:nodelist, '$MIGA_TEST_NODELIST')
     end
     ENV['MIGA_TEST_NODELIST'] = f
     capture_stderr { d1.runopts(:nodelist, '$MIGA_TEST_NODELIST') }
-    helper_daemon_launch_job
+    helper_daemon_launch_job(1)
     assert_equal("project1:p:miga-project\n", File.read(t))
   end
 
   def test_launch_job_qsub
-    $d1.runopts(:type, 'qsub')
-    $d1.runopts(:cmd, 'echo {{task_name}}')
+    daemon.runopts(:type, 'qsub')
+    daemon.runopts(:cmd, 'echo {{task_name}}')
     helper_daemon_launch_job
-    assert_equal('project1:p:miga-project', $d1.jobs_running.first[:pid])
+    assert_equal('project0:p:miga-project', daemon.jobs_running.first[:pid])
   end
 
   def test_launch_job_failure
-    d1 = $d1
+    d1 = daemon(1)
     d1.runopts(:type, 'qsub')
     d1.runopts(:cmd, 'echo ""')
-    helper_datasets_with_results.first.inactivate!
+    helper_datasets_with_results(1, 1).first.inactivate!
     capture_stderr { d1.check_project }
-    omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
+
+    declare_forks
     out = capture_stderr { d1.launch_job(d1.jobs_to_run.shift) }.string
     assert_match(/Unsuccessful project1:p:miga-project, rescheduling/, out)
     assert_equal(0, d1.jobs_running.size)
     assert_equal(1, d1.jobs_to_run.size)
   end
 
-  def helper_daemon_launch_job
-    omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
-    d1 = $d1
+  def test_verbosity
+    d1 = daemon
+    d1.runopts(:verbosity, 0)
+    out = capture_stderr { d1.in_loop }.string
+    assert_empty(out)
+
+    d1.runopts(:verbosity, 1)
     helper_datasets_with_results.first.inactivate!
-    assert_equal(0, d1.jobs_to_run.size, 'The queue should be empty')
-    capture_stderr { d1.check_project }
-    assert_equal(1, d1.jobs_to_run.size, 'The queue should have one job')
-    capture_stderr { d1.flush! }
-    sleep(1)
-    assert_equal(0, d1.jobs_to_run.size, 'There should be nothing running')
-    assert_equal(1, d1.jobs_running.size, 'There should be one job running')
-  end
+    out = capture_stderr { d1.check_project }
+    assert_match(/Queueing miga-project:p/, out.string)
 
+    d1.runopts(:verbosity, 2)
+    out = capture_stderr { d1.in_loop }.string
+    assert_match(/Reloading project/, out)
+
+    d1.runopts(:verbosity, 3)
+    out = capture_stderr { d1.in_loop }.string
+    assert_match(/Daemon loop start/, out)
+  end
 end