miga-base 0.6.4.2 → 0.7.0.0

data/lib/miga/daemon/base.rb

@@ -1,6 +1,4 @@
 
-require 'daemons'
-require 'date'
 require 'shellwords'
 
 class MiGA::Daemon < MiGA::MiGA
@@ -55,21 +53,5 @@ module MiGA::Daemon::Base
   # complete
   def shutdown_when_done?() !!runopts(:shutdown_when_done); end
 
-  ##
-  # Initializes the daemon with +opts+
-  def start(opts = []) daemon('start', opts); end
-
-  ##
-  # Stops the daemon with +opts+
-  def stop(opts = []) daemon('stop', opts); end
-
-  ##
-  # Restarts the daemon with +opts+
-  def restart(opts = []) daemon('restart', opts); end
-
-  ##
-  # Returns the status of the daemon with +opts+.
-  def status(opts = []) daemon('status', opts); end
-
 end
 

data/lib/miga/lair.rb

@@ -1,20 +1,14 @@
 # @package MiGA
 # @license Artistic-2.0
 
-require 'daemons'
 require 'miga/daemon'
+require 'miga/common/with_daemon'
 
 ##
 # Lair of MiGA Daemons handling job submissions
 class MiGA::Lair < MiGA::MiGA
-  ##
-  # When was the last time the chief daemon in this lair was seen active?
-  # Returns Time
-  def self.last_alive(path)
-    f = File.expand_path('.lair-alive', path)
-    return nil unless File.exist? f
-    Time.parse(File.read(f))
-  end
+  include MiGA::Common::WithDaemon
+  extend MiGA::Common::WithDaemonClass
 
   # Absolute path to the directory where the projects are located
   attr_reader :path
@@ -22,69 +16,81 @@ class MiGA::Lair < MiGA::MiGA
   # Options used to setup the chief daemon
   attr_accessor :options
 
-  # Integer indicating the current iteration
-  attr_reader :loop_i
-
   ##
   # Initialize an inactive daemon for the directory at +path+. See #daemon
   # to wake the chief daemon. Supported options include:
   # - json: json definition for all children daemons, by default: nil
   # - latency: time to wait between iterations in seconds, by default: 120
   # - wait_for: time to wait for a daemon to report being alive in seconds,
-  #   by default: 900
+  #   by default: 30
   # - keep_inactive: boolean indicating if daemons should stay alive even when
   #   inactive (when all tasks are complete), by default: false
   # - name: A name for the chief daemon process, by default: basename of +path+
+  # - trust_timestamp: boolean indicating if the +modified+ timestamp of the
+  #   project is to be trusted to determine changes in the project,
+  #   by default: true
+  # - dry: Only report when daemons would be launched, but don't actually launch
+  #   them
   def initialize(path, opts = {})
     @path = File.expand_path(path)
     @options = opts
-    @loop_i = -1
     {
       json: nil,
-      latency: 120,
-      wait_for: 900,
+      latency: 30,
+      wait_for: 30,
       keep_inactive: false,
-      name: File.basename(@path)
+      trust_timestamp: true,
+      name: File.basename(@path),
+      dry: false
     }.each { |k, v| @options[k] = v if @options[k].nil? }
   end
 
   ##
-  # When was the last time the chief daemon for was seen active here?
-  # Returns Time.
-  def last_alive
-    MiGA::Lair.last_alive path
+  # Path to the lair's chief daemon's home
+  alias daemon_home path
+
+  ##
+  # Name of the lair's chief daemon
+  def daemon_name
+    "MiGA:#{options[:name]}"
+  end
+
+  ##
+  # First loop of the lair's chief daemon
+  def daemon_first_loop
+    say '-----------------------------------'
+    say '%s launched' % daemon_name
+    say '-----------------------------------'
+    say 'Configuration options:'
+    say options.to_s
   end
 
   ##
-  # Returns Hash containing the default options for the daemon.
-  def default_options
-    { dir_mode: :normal, dir: path, multiple: false, log_output: true }
+  # Run one loop step. Returns a Boolean indicating if the loop should continue.
+  def daemon_loop
+    check_directories
+    return false if options[:dry]
+    sleep(options[:latency])
+    true
   end
 
   ##
-  # Launches the +task+ with options +opts+ (as command-line arguments) and
-  # returns the process ID as an Integer. If +wait+ it waits for the process to
-  # complete, immediately returns otherwise.
-  # Supported tasks: start, stop, restart, status.
-  def daemon(task, opts = [], wait = true)
-    MiGA.DEBUG "Lair.daemon #{task} #{opts}"
-    config = default_options
-    opts.unshift(task.to_s)
-    config[:ARGV] = opts
-    # This additional degree of separation below was introduced so the Daemons
-    # package doesn't kill the parent process in workflows.
-    pid = fork do
-      Daemons.run_proc("MiGA:#{options[:name]}", config) { while in_loop; end }
+  # Terminate all daemons in the lair (including the chief daemon)
+  def terminate_daemons
+    terminate_daemon(self)
+    each_project do |project|
+      terminate_daemon(MiGA::Daemon.new(project))
     end
-    Process.wait(pid) if wait
-    pid
   end
 
   ##
-  # Tell the world that you're alive.
-  def declare_alive
-    File.open(File.join(path, '.lair-alive'), 'w') do |fh|
-      fh.print Time.now.to_s
+  # Send termination message to +daemon+, an object implementing
+  # +MiGA::Common::WithDaemon+
+  def terminate_daemon(daemon)
+    say "Probing #{daemon.class} #{daemon.daemon_home}"
+    if daemon.active?
+      say 'Sending termination message'
+      FileUtils.touch(daemon.terminate_file)
     end
   end
 
@@ -94,13 +100,14 @@ class MiGA::Lair < MiGA::MiGA
   # Searches for MiGA projects recursively in all
   # subdirectories that are not MiGA projects.
   def each_project(dir = path)
-    Dir.children(dir).each do |f|
+    Dir.entries(dir).each do |f|
+      next if %w[. ..].include?(f) # Ruby <= 2.3 doesn't have Dir.children
       f = File.join(dir, f)
       if MiGA::Project.exist? f
         project = MiGA::Project.load(f)
         raise "Cannot load project: #{f}" if project.nil?
         yield(project)
-      elsif Dir.exists? f
+      elsif Dir.exist? f
         each_project(f) { |project| yield(project) }
       end
     end
@@ -110,8 +117,13 @@ class MiGA::Lair < MiGA::MiGA
   # Traverse directories checking MiGA projects
   def check_directories
     each_project do |project|
-      alive = MiGA::Daemon.last_alive(project)
-      next if !alive.nil? && alive > Time.now - options[:wait_for]
+      d = MiGA::Daemon.new(project)
+      next if d.active?
+      l_alive = d.last_alive
+      unless l_alive.nil?
+        next if options[:trust_timestamp] && project.metadata.updated < l_alive
+        next if l_alive > Time.now - options[:wait_for]
+      end
       launch_daemon(project)
     end
   end
@@ -122,32 +134,9 @@ class MiGA::Lair < MiGA::MiGA
     say "Launching daemon: #{project.path}"
     d = MiGA::Daemon.new(project, options[:json])
     d.runopts(:shutdown_when_done, true) unless options[:keep_inactive]
-    d.daemon(:start, [], false)
-  end
-
-  ##
-  # Run one loop step. Returns a Boolean indicating if the loop should continue.
-  def in_loop
-    declare_alive
-    if loop_i == -1
-      say '-----------------------------------'
-      say 'MiGA:%s launched' % options[:name]
-      say '-----------------------------------'
-      say 'Configuration options:'
-      say options.to_s
-      @loop_i = 0
+    unless options[:dry]
+      d.start
+      sleep(1) # <- to make sure the daemon started up (it takes about 0.1 secs)
     end
-    @loop_i += 1
-    check_directories
-    sleep(options[:latency])
-    true
-  end
-
-  ##
-  # Terminates a chief daemon
-  def terminate
-    say 'Terminating chief daemon...'
-    f = File.expand_path('.lair-alive', project.path)
-    File.unlink(f) if File.exist? f
   end
 end

data/lib/miga/metadata.rb

@@ -135,6 +135,18 @@ class MiGA::Metadata < MiGA::MiGA
     data.each { |k, v| blk.call(k, v) }
   end
 
+  ##
+  # Time of last update
+  def updated
+    Time.parse(self[:updated]) unless self[:updated].nil?
+  end
+
+  ##
+  # Time of creation
+  def created
+    Time.parse(self[:created]) unless self[:created].nil?
+  end
+
   ##
   # Show contents in JSON format as a String
   def to_json

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.6, 4, 2]
+  VERSION = [0.7, 0, 0]
 
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 4, 15)
+  VERSION_DATE = Date.new(2020, 4, 18)
 
   ##
   # Reference of MiGA.

data/test/common_test.rb

@@ -33,7 +33,7 @@ class CommonTest < Test::Unit::TestCase
     err = capture_stderr do
       MiGA::MiGA.DEBUG 'Dandadi'
     end
-    assert(err.string =~ /Dandadi\n    .*block in test_debug_trace/)
+    assert_match(/Dandadi\n    .*block in test_debug_trace/, err.string)
     MiGA::MiGA.DEBUG_TRACE_OFF
     err = capture_stderr do
       MiGA::MiGA.DEBUG 'Dandada'
@@ -66,7 +66,7 @@ class CommonTest < Test::Unit::TestCase
     assert_raise do
       File.generic_transfer(hello, world, :monkey)
     end
-    assert(!File.exist?(world), 'A monkey shouldn\'t create files.')
+    assert_path_not_exist(world, 'A monkey shouldn\'t create files.')
   ensure
     FileUtils.rm_rf $tmp
   end
@@ -75,8 +75,8 @@ class CommonTest < Test::Unit::TestCase
     assert_equal('Xa sp. C', 'Xa_sp__C'.unmiga_name)
     assert_equal('X_______', 'X^*.!{}!'.miga_name)
     assert_equal('aB09', 'aB09'.miga_name)
-    assert('R2D2'.miga_name?)
-    assert(!'C3-PO'.miga_name?)
+    assert_predicate('R2D2', :miga_name?)
+    assert_not_predicate('C3-PO', :miga_name?)
     assert_equal("123\n1\n", '1231'.wrap_width(3))
   end
 

data/test/daemon_test.rb

@@ -42,8 +42,8 @@ class DaemonTest < Test::Unit::TestCase
   def test_check_project
     d1 = $d1
     helper_datasets_with_results.first.inactivate!
-    out = capture_stdout { d1.check_project }
-    assert(out.string =~ /Queueing miga-project:p/)
+    out = capture_stderr { d1.check_project }
+    assert_match(/Queueing miga-project:p/, out.string)
     assert_equal(1, d1.jobs_to_run.size)
     assert_equal(:p, d1.jobs_to_run.first[:job])
     assert_equal('project1:p:miga-project', d1.get_job(:p)[:task_name])
@@ -53,10 +53,10 @@ class DaemonTest < Test::Unit::TestCase
     p = $p1
     d = $d1
     d.runopts(:maxjobs, 0, true)
-    assert(d.jobs_to_run.empty?)
+    assert_empty(d.jobs_to_run)
     ds = p.add_dataset('ds1')
     d.check_datasets
-    assert(d.jobs_to_run.empty?)
+    assert_empty(d.jobs_to_run)
     FileUtils.cp(
       File.expand_path('daemon/daemon.json', p.path),
       File.expand_path('data/01.raw_reads/ds1.1.fastq', p.path)
@@ -66,10 +66,10 @@ class DaemonTest < Test::Unit::TestCase
       File.expand_path('data/01.raw_reads/ds1.done', p.path)
     )
     ds.first_preprocessing(true)
-    out = capture_stdout do
+    out = capture_stderr do
       d.check_datasets
     end
-    assert(out.string =~ /Queueing #{ds.name}:d/)
+    assert_match(/Queueing #{ds.name}:d/, out.string)
     assert_equal(1, d.jobs_to_run.size)
     assert_equal('echo project1:d:ds1 >/dev/null', d.jobs_to_run.first[:cmd])
     assert_equal(d.jobs_to_run.first, d.get_job(:d, ds))
@@ -79,15 +79,15 @@ class DaemonTest < Test::Unit::TestCase
     p = $p1
     d = $d1
     d.runopts(:latency, 0, true)
-    assert_equal(-1, d.loop_i)
+    assert_nil(d.loop_i)
     assert_nil(d.last_alive)
-    out = capture_stdout { d.in_loop }
+    out = capture_stderr { d.in_loop }.string
     assert_equal(Time, d.last_alive.class)
-    assert(out.string =~ /-{20}\n.*MiGA:#{p.name} launched/)
-    10.times{ d.in_loop }
+    assert_match(/-{20}\n.*MiGA:#{p.name} launched/, out)
+    11.times { d.in_loop }
     assert_equal(11, d.loop_i)
-    out = capture_stdout { d.in_loop }
-    assert(out.string =~ /Probing running jobs/)
+    out = capture_stderr { d.in_loop }.string
+    assert_match(/Probing running jobs/, out)
     assert_equal(0, d.loop_i)
   end
 
@@ -98,25 +98,24 @@ class DaemonTest < Test::Unit::TestCase
     assert_equal(0, d.latency)
     omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
     child = $child = d.daemon(:start, ['--shush'])
-    assert(child.is_a? Integer)
-    assert(child != 0, 'The daemond process should have non-zero PID')
+    assert_not_nil(child)
+    assert_gt(child, 1)
     assert_equal(0, `ps -p "#{child}" -o ppid=`.strip.to_i,
       'The daemon process should be detached')
     sleep(3)
-    dpath = File.join(p.path, 'daemon', "MiGA:#{p.name}")
-    assert(File.exist?("#{dpath}.pid"))
-    out = capture_stdout { d.stop }
+    assert_path_exist(d.pid_file)
+    out = capture_stderr { d.stop }.string
     assert_raise(Errno::ESRCH) { Process.kill(0, child) }
-    assert_equal('', out.string)
-    assert(!File.exist?("#{dpath}.pid"))
-    assert(File.exist?("#{dpath}.output"))
-    File.open("#{dpath}.output", "r") do |fh|
-      l = fh.each_line.to_a
-      assert(l[0] =~ /-{20}\n/)
-      assert(l[1] =~ /MiGA:#{p.name} launched/)
-      assert(l[2] =~ /-{20}\n/)
-      assert(l[5] =~ /Probing running jobs\n/)
-    end
+    assert_match(/Sending termination message/, out)
+    assert_path_not_exist(d.pid_file)
+    assert_path_exist(d.output_file)
+    l = File.readlines(d.output_file)
+    {
+      0 => /-{20}\n/,
+      1 => /MiGA:#{p.name} launched/,
+      2 => /-{20}\n/,
+      5 => /Probing running jobs\n/
+    }.each { |k, v| assert_match(v, l[k], "unexpected line: #{k}") }
   ensure
     begin
       Process.kill('KILL', $child) if !$child.nil?
@@ -129,8 +128,9 @@ class DaemonTest < Test::Unit::TestCase
     p = MiGA::Project.new(File.expand_path('last_alive', $tmp))
     d = MiGA::Daemon.new(p)
     assert_nil(d.last_alive)
+    omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
     d.declare_alive
-    assert(d.last_alive - Time.now < 1)
+    assert_lt(d.last_alive, Time.now)
   end
 
   def test_options
@@ -150,17 +150,28 @@ class DaemonTest < Test::Unit::TestCase
   end
 
   def test_say
-    out = capture_stdout { $d1.say 'Olm' }
-    assert(out.string =~ /^\[.*\] Olm/)
+    out = capture_stderr { $d1.say 'Olm' }.string
+    assert_match(/^\[.*\] Olm/, out)
   end
 
   def test_terminate
     d = $d1
+    assert_not_predicate(d, :active?)
+    assert_path_not_exist(d.alive_file)
+    assert_path_not_exist(d.terminated_file)
+
+    omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
     d.declare_alive
+    assert_predicate(d, :active?)
+    assert_not_nil(d.last_alive)
+    assert_path_exist(d.alive_file)
+    assert_path_not_exist(d.terminated_file)
+
+    d.terminate
+    assert_path_not_exist(d.alive_file)
+    assert_path_exist(d.terminated_file)
+    assert_not_predicate(d, :active?)
     assert_not_nil(d.last_alive)
-    out = capture_stdout { d.terminate }
-    assert(out.string =~ /Terminating daemon/)
-    assert_nil(d.last_alive)
   end
 
   def test_maxjobs_json
@@ -168,7 +179,7 @@ class DaemonTest < Test::Unit::TestCase
     helper_datasets_with_results(3)
     assert_equal(0, d1.jobs_running.size)
     assert_equal(0, d1.jobs_to_run.size)
-    capture_stdout { d1.in_loop }
+    capture_stderr { d1.in_loop }
     assert_equal(1, d1.jobs_running.size)
     assert_equal(2, d1.jobs_to_run.size)
   end
@@ -179,7 +190,7 @@ class DaemonTest < Test::Unit::TestCase
     d1.runopts(:maxjobs, 2)
     assert_equal(0, d1.jobs_running.size)
     assert_equal(0, d1.jobs_to_run.size)
-    capture_stdout { d1.in_loop }
+    capture_stderr { d1.in_loop }
     assert_equal(2, d1.jobs_running.size)
     assert_equal(1, d1.jobs_to_run.size)
   end
@@ -195,9 +206,9 @@ class DaemonTest < Test::Unit::TestCase
     File.open(f, 'w') do |h|
       h.puts '{"jobs_running":[{"ds":1,"ds_name":"d1"},{}],"jobs_to_run":[]}'
     end
-    out = capture_stdout { d1.load_status }
+    out = capture_stderr { d1.load_status }.string
     assert_equal(2, d1.jobs_running.size)
-    assert(out.string =~ /Loading previous status/)
+    assert_match(/Loading previous status/, out)
     assert_equal(MiGA::Dataset, d1.jobs_running[0][:ds].class)
     assert_nil(d1.jobs_running[1][:ds])
   end
@@ -215,10 +226,10 @@ class DaemonTest < Test::Unit::TestCase
         '{"job":"trimmed_reads","ds":1,"ds_name":"d0"}]' \
         ',"jobs_to_run":[]}'
     end
-    capture_stdout { d1.load_status }
+    capture_stderr { d1.load_status }
     assert_equal(3, d1.jobs_running.size)
-    out = capture_stdout { d1.flush! }
-    assert(out.string =~ /Completed pid:/)
+    out = capture_stderr { d1.flush! }.string
+    assert_match(/Completed pid:/, out)
     assert_equal([], d1.jobs_running)
   end
 
@@ -227,8 +238,8 @@ class DaemonTest < Test::Unit::TestCase
     f = File.join($tmp, 'nodes.txt')
     File.open(f, 'w') { |h| h.puts 'localhost' }
     assert_equal(true, d1.next_host)
-    out = capture_stdout { d1.runopts(:nodelist, f) }
-    assert(out.string =~ /Reading node list:/)
+    out = capture_stderr { d1.runopts(:nodelist, f) }.string
+    assert_match(/Reading node list:/, out)
     assert_equal(true, d1.next_host)
     d1.runopts(:type, 'ssh')
     assert_equal(0, d1.next_host)
@@ -236,14 +247,14 @@ class DaemonTest < Test::Unit::TestCase
     File.open(f, 'w') do |h|
       h.puts '{"jobs_running":[{"job":"p","hostk":0}], "jobs_to_run":[]}'
     end
-    capture_stdout { d1.load_status }
+    capture_stderr { d1.load_status }
     assert_nil(d1.next_host)
   end
 
   def test_shutdown_when_done
     $d1.runopts(:shutdown_when_done, true)
-    out = capture_stdout { assert(!$d1.in_loop) }
-    assert(out.string =~ /Nothing else to do/)
+    out = capture_stderr { assert { !$d1.in_loop } }.string
+    assert_match(/Nothing else to do/, out)
   end
 
   def test_update_format_0
@@ -276,7 +287,7 @@ class DaemonTest < Test::Unit::TestCase
       d1.runopts(:nodelist, '$MIGA_TEST_NODELIST')
     end
     ENV['MIGA_TEST_NODELIST'] = f
-    capture_stdout { d1.runopts(:nodelist, '$MIGA_TEST_NODELIST') }
+    capture_stderr { d1.runopts(:nodelist, '$MIGA_TEST_NODELIST') }
     helper_daemon_launch_job
     assert_equal("project1:p:miga-project\n", File.read(t))
   end
@@ -293,20 +304,22 @@ class DaemonTest < Test::Unit::TestCase
     d1.runopts(:type, 'qsub')
     d1.runopts(:cmd, 'echo ""')
     helper_datasets_with_results.first.inactivate!
-    capture_stdout { d1.check_project }
-    out = capture_stdout { d1.launch_job(d1.jobs_to_run.shift) }
-    assert(out.string =~ /Unsuccessful project1:p:miga-project, rescheduling/)
+    capture_stderr { d1.check_project }
+    omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
+    out = capture_stderr { d1.launch_job(d1.jobs_to_run.shift) }.string
+    assert_match(/Unsuccessful project1:p:miga-project, rescheduling/, out)
     assert_equal(0, d1.jobs_running.size)
     assert_equal(1, d1.jobs_to_run.size)
   end
 
   def helper_daemon_launch_job
+    omit_if($jruby_tests, 'JRuby doesn\'t implement fork.')
     d1 = $d1
     helper_datasets_with_results.first.inactivate!
     assert_equal(0, d1.jobs_to_run.size, 'The queue should be empty')
-    capture_stdout { d1.check_project }
+    capture_stderr { d1.check_project }
     assert_equal(1, d1.jobs_to_run.size, 'The queue should have one job')
-    capture_stdout { d1.flush! }
+    capture_stderr { d1.flush! }
     sleep(1)
     assert_equal(0, d1.jobs_to_run.size, 'There should be nothing running')
     assert_equal(1, d1.jobs_running.size, 'There should be one job running')