miga-base 0.6.4.2 → 0.7.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4b861cfbdaa0d9cd28403345bd4dc61c4814a07c09a59709f09415dd05007a60
-  data.tar.gz: 4738c33442f61be5a6fe0c6cf36c05cabb4fcd5b08f481099651bdacbc2d97fc
+  metadata.gz: 209e314b880d88c662aa9fbb04d62df7b4676e5d6af885e0cd23cea354accdec
+  data.tar.gz: d7fd5f25b2132880374693773e4cd1d58a331aa34d06ba31be53a5926e47f0e0
 SHA512:
-  metadata.gz: ad69cb41e41f11d5aa0ad38af53a007eac213ab5901df9432d140d510fda2d8b8a7e6f4fda5302709e47c0150030cfdf601c54d9ce308d99aa82deeeb05ed0e8
-  data.tar.gz: aba5401cef4cafba7afdf48765ab3690313d768140229fcb297b1d02eba441816fc0f8f69ecfa7e477de6ac0d5387c324e900fb6b20151bd736bfe7814737155
+  metadata.gz: d2e34541cd0191e36524aa1f0811fe3933837cdbc2fb550ced6692423ca873bc900b188ddd852a18839f3bb47a13a19cf9e9de60e65ed1b11c112f0bc217996d
+  data.tar.gz: 1dd8e5a74546e47271fb0f35a983234364ce1e855d0ab6194c3782dabc5a82920504c99ca513409b7e6c60244fc9320c2ff88c0f041f72cbc83733332e605f39

data/lib/miga/cli/action/daemon.rb

@@ -11,13 +11,10 @@ class MiGA::Cli::Action::Daemon < MiGA::Cli::Action
     cli.expect_operation = true
     cli.parse do |opt|
       opt.separator 'Available operations:'
-      { start:   'Start an instance of the application.',
-        stop:    'Start an instance of the application.',
-        restart: 'Stop all instances and restart them afterwards.',
-        reload:  'Send a SIGHUP to all instances of the application.',
-        run:     'Start the application and stay on top.',
-        zap:     'Set the application to a stopped state.',
-        status:  'Show status (PID) of application instances.'
+      { start:   'Start an instance of the application',
+        stop:    'Start an instance of the application',
+        run:     'Start the application and stay on top',
+        status:  'Show status (PID) of application instances'
       }.each { |k,v| opt.separator sprintf '    %*s%s', -33, k, v }
       opt.separator ''
 

data/lib/miga/cli/action/lair.rb

@@ -11,13 +11,11 @@ class MiGA::Cli::Action::Lair < MiGA::Cli::Action
     cli.expect_operation = true
     cli.parse do |opt|
       opt.separator 'Available operations:'
-      { start:   'Start an instance of the application.',
-        stop:    'Start an instance of the application.',
-        restart: 'Stop all instances and restart them afterwards.',
-        reload:  'Send a SIGHUP to all instances of the application.',
-        run:     'Start the application and stay on top.',
-        zap:     'Set the application to a stopped state.',
-        status:  'Show status (PID) of application instances.'
+      { start:   'Start an instance of the application',
+        stop:    'Start an instance of the application',
+        run:     'Start the application and stay on top',
+        status:  'Show status (PID) of application instances',
+        terminate: 'Terminate all daemons in the lair and exit'
       }.each { |k,v| opt.separator sprintf '    %*s%s', -33, k, v }
       opt.separator ''
 
@@ -26,6 +24,10 @@ class MiGA::Cli::Action::Lair < MiGA::Cli::Action
         '-p', '--path PATH',
         '(Mandatory) Path to the directory where the MiGA projects are located'
       ) { |v| cli[:path] = v }
+      opt.on(
+        '--json PATH',
+        'Path to a custom daemon definition in json format'
+      ) { |v| cli[:json] = v }
       opt.on(
         '--latency INT', Integer,
         'Time to wait between iterations in seconds, by default: 120'
@@ -33,21 +35,25 @@ class MiGA::Cli::Action::Lair < MiGA::Cli::Action
       opt.on(
         '--wait-for INT', Integer,
         'Time to wait for a daemon to report being alive in seconds',
-        'by default: 900'
+        'by default: 30'
       ) { |v| cli[:wait_for] = v }
       opt.on(
         '--keep-inactive',
         'If set, daemons are kept alive even when inactive;',
         'i.e., when all tasks are complete'
       ) { |v| cli[:keep_inactive] = v }
+      opt.on(
+        '--no-trust-timestamp',
+        'Check all results instead of trusting project timestamps'
+      ) { |v| cli[:trust_timestamp] = v }
       opt.on(
         '--name STRING',
         'A name for the chief daemon process'
       ) { |v| cli[:name] = v }
       opt.on(
-        '--json PATH',
-        'Path to a custom daemon definition in json format'
-      ) { |v| cli[:json] = v }
+        '--dry',
+        'Report when daemons would be launched, but don\'t actually launch them'
+      ) { |v| cli[:dry] = v }
       cli.opt_common(opt)
 
       opt.separator 'Daemon options:'
@@ -73,9 +79,13 @@ class MiGA::Cli::Action::Lair < MiGA::Cli::Action
 
   def perform
     cli.ensure_par(path: '-p')
-    k_opts = %i[latency wait_for keep_inactive name json]
-    opts = Hash[k_opts.map { |k| [k, cli[k]] }]
-    lair = MiGA::Lair.new(cli[:path], opts)
-    lair.daemon(cli.operation, cli[:daemon_opts])
+    if cli.operation.to_sym == :terminate
+      MiGA::Lair.new(cli[:path]).terminate_daemons
+    else
+      k_opts = %i[json latency wait_for keep_inactive trust_timestamp name dry]
+      opts = Hash[k_opts.map { |k| [k, cli[k]] }]
+      lair = MiGA::Lair.new(cli[:path], opts)
+      lair.daemon(cli.operation, cli[:daemon_opts])
+    end
   end
 end

data/lib/miga/common/with_daemon.rb

@@ -0,0 +1,203 @@
+
+require 'daemons'
+require 'miga/common/with_daemon_class'
+
+##
+# Helper module with specific functions to handle objects that have daemons.
+# The class including it must +extend MiGA::Common::WithDaemonClass+ and define:
+# - +#daemon_home+ Path to the daemon's home
+# - +#daemon_name+ Name of the daemon
+# - +#daemon_loop+ One loop of the daemon to be repeatedly called
+# - +#daemon_first_loop+ To be executed before the first call to +#daemon_loop+
+module MiGA::Common::WithDaemon
+  # Process ID of the forked process declaring the daemon alive
+  attr :declare_alive_pid
+
+  # Loop counter
+  attr :loop_i
+
+  def pid_file
+    File.join(daemon_home, "#{daemon_name}.pid")
+  end
+
+  def output_file
+    File.join(daemon_home, "#{daemon_name}.output")
+  end
+  
+  def terminate_file
+    File.join(daemon_home, 'terminate-daemon')
+  end
+
+  def alive_file
+    self.class.alive_file(daemon_home)
+  end
+
+  def terminated_file
+    self.class.terminated_file(daemon_home)
+  end
+
+  ##
+  # When was the daemon last seen active?
+  def last_alive
+    self.class.last_alive(daemon_home)
+  end
+
+  ##
+  # Is the daemon active?
+  def active?
+    return false unless File.exist? alive_file
+    last_alive > Time.now - 60
+  end
+
+  ##
+  # Tell the world that you're alive.
+  def declare_alive
+    if active?
+      raise "Trying to declare alive an active daemon, if you think this is a" \
+        " mistake please remove #{alive_file} or try again in 1 minute"
+    end
+    @declare_alive_pid = fork { declare_alive_loop }
+    sleep(1) # <- to wait for the process check
+  end
+
+  ##
+  # Loop checking if the process with PID +pid+ is still alive.
+  # By default, the parent process.
+  # Do not use directly, use +declare_alive+ instead.
+  # Returns a symbol indicating the reason to stop:
+  # - +:no_home+ Daemon's home does not exist
+  # - +:no_process_alive+ Process is not currently running
+  # - +:termination_file+ Found termination file
+  def declare_alive_loop(pid = Process.ppid)
+    i = -1
+    loop do
+      i += 1
+      return :no_home unless Dir.exist? daemon_home
+      return :no_process_alive unless process_alive? pid
+      write_alive_file if i % 30 == 0
+      return :termination_file if termination_file? pid
+      sleep(1)
+    end
+  end
+
+  def write_alive_file
+    File.open(alive_file, 'w') { |fh| fh.print Time.now.to_s }
+  end
+
+  ##
+  # Check if the process with PID +pid+ is still alive,
+  # call +terminate+ otherwise.
+  def process_alive?(pid)
+    Process.kill(0, pid)
+    true
+  rescue Errno::ESRCH, Errno::EPERM, Errno::ENOENT
+    terminate
+    false
+  end
+
+  ##
+  # Check if a termination file exists and terminate process with PID +pid+
+  # if it does. Do not kill any process if +pid+ is +nil+
+  def termination_file?(pid)
+    return false unless File.exist? terminate_file
+    say 'Found termination file, terminating'
+    File.unlink(terminate_file)
+    terminate
+    Process.kill(9, pid) unless pid.nil?
+    true
+  end
+
+  ##
+  # Returns Hash containing the default options for the daemon.
+  def default_options
+    {
+      dir_mode: :normal, dir: daemon_home, multiple: false, log_output: true,
+      stop_proc: :terminate
+    }
+  end
+
+  ##
+  # Launches the +task+ with options +opts+ (as command-line arguments) and
+  # returns the process ID as an Integer. If +wait+ it waits for the process to
+  # complete, immediately returns otherwise.
+  # Supported tasks: start, run, stop, status.
+  def daemon(task, opts = [], wait = true)
+    MiGA::MiGA.DEBUG "#{self.class}#daemon #{task} #{opts}"
+    task = task.to_sym
+    raise "Unsupported task: #{task}" unless respond_to? task
+    return send(task, opts, wait) unless %i[start run].include? task
+
+    # start & run:
+    options = default_options
+    opts.unshift(task.to_s)
+    options[:ARGV] = opts
+    # This additional degree of separation below was introduced so the Daemons
+    # package doesn't kill the parent process in workflows.
+    pid = fork { launch_daemon_proc(options) }
+    Process.wait(pid) if wait
+    pid
+  end
+
+  ##
+  # Stops the daemon with +opts+
+  def stop(opts = [], wait = true)
+    if active?
+      say 'Sending termination message'
+      FileUtils.touch(terminate_file)
+      sleep(0.5) while active? if wait
+      File.unlink(pid_file) if File.exist?(pid_file)
+    else
+      say 'No running instances'
+    end
+  end
+
+  ##
+  # Returns the status of the daemon with +opts+
+  def status(opts = [], wait = true)
+    if active?
+      say "Running with pid #{File.read(pid_file)}"
+    else
+      say 'Not running'
+    end
+  end
+
+  ##
+  # Pass daemon options to +Daemons+. Do not use directly, use +daemon+ instead.
+  def launch_daemon_proc(options)
+    Daemons.run_proc("#{daemon_name}", options) { while in_loop; end }
+  end
+
+  ##
+  # Initializes the daemon with +opts+
+  def start(opts = [], wait = true)
+    daemon(:start, opts, wait)
+  end
+
+  ##
+  # Initializes the daemon on top with +opts+
+  def run(opts = [], wait = true)
+    daemon(:run, opts, wait)
+  end
+
+  ##
+  # One loop, returns a boolean indicating if the execution should continue
+  def in_loop
+    if loop_i.nil?
+      declare_alive
+      daemon_first_loop
+      @loop_i = -1
+    end
+    @loop_i += 1
+    daemon_loop
+  end
+
+  ##
+  # Declares a daemon termination. Do not use, directly, use #stop instead.
+  def terminate
+    unless declare_alive_pid.nil?
+      Process.kill(9, declare_alive_pid)
+      @declare_alive_pid = nil
+    end
+    File.rename(alive_file, terminated_file) if File.exist? alive_file
+  end
+end

data/lib/miga/common/with_daemon_class.rb

@@ -0,0 +1,32 @@
+
+##
+# Helper module with specific class-level functions to be used with
+# +include MiGA::Common::WithDaemon+.
+module MiGA::Common::WithDaemonClass
+  ##
+  # Path to the daemon home from the parent's +path+
+  def daemon_home(path)
+    path
+  end
+
+  ##
+  # Path to the alive file
+  def alive_file(path)
+    File.join(daemon_home(path), '.daemon-alive')
+  end
+
+  ##
+  # Path to the terminated file
+  def terminated_file(path)
+    File.join(daemon_home(path), '.daemon-terminated')
+  end
+
+  ##
+  # When was a daemon last seen at +path+?
+  def last_alive(path)
+    f = alive_file(path)
+    f = terminated_file(path) unless File.exist? f
+    return nil unless File.exist? f
+    Time.parse(File.read(f))
+  end
+end

data/lib/miga/daemon.rb

@@ -2,6 +2,7 @@
 # @license Artistic-2.0
 
 require 'miga/project'
+require 'miga/common/with_daemon'
 require 'miga/daemon/base'
 
 ##
@@ -9,29 +10,30 @@ require 'miga/daemon/base'
 class MiGA::Daemon < MiGA::MiGA
 
   include MiGA::Daemon::Base
+  include MiGA::Common::WithDaemon
+  extend MiGA::Common::WithDaemonClass
 
-  ##
-  # When was the last time a daemon for the MiGA::Project +project+ was seen
-  # active? Returns Time.
-  def self.last_alive(project)
-    f = File.expand_path('daemon/alive', project.path)
-    return nil unless File.exist? f
-    Time.parse(File.read(f))
+  class << self
+    ##
+    # Daemon's home inside the MiGA::Project +project+ or a String with the
+    # full path to the project's 'daemon' folder
+    def daemon_home(project)
+      return project if project.is_a? String
+      File.join(project.path, 'daemon')
+    end
   end
 
-  # Array of all spawned daemons.
-  $_MIGA_DAEMON_LAIR = []
-
   # MiGA::Project in which the daemon is running
   attr_reader :project
+
   # Options used to setup the daemon
   attr_reader :options
+
   # Array of jobs next to be executed
   attr_reader :jobs_to_run
+
   # Array of jobs currently running
   attr_reader :jobs_running
-  # Integer indicating the current iteration
-  attr_reader :loop_i
 
   ##
   # Initialize an unactive daemon for the MiGA::Project +project+. See #daemon
@@ -40,72 +42,74 @@ class MiGA::Daemon < MiGA::MiGA
   # is used. In either case, missing variables are used as defined in
   # ~/.miga_daemon.json.
   def initialize(project, json = nil)
-    $_MIGA_DAEMON_LAIR << self
     @project = project
     @runopts = {}
-    json ||= File.expand_path('daemon/daemon.json', project.path)
+    json ||= File.join(project.path, 'daemon/daemon.json')
     MiGA::Json.parse(
       json, default: File.expand_path('.miga_daemon.json', ENV['MIGA_HOME'])
     ).each { |k,v| runopts(k, v) }
     update_format_0
     @jobs_to_run = []
     @jobs_running = []
-    @loop_i = -1
   end
 
   ##
-  # When was the last time a daemon for the current project was seen active?
-  # Returns Time.
-  def last_alive
-    MiGA::Daemon.last_alive project
+  # Path to the daemon home
+  def daemon_home
+    self.class.daemon_home(project)
   end
 
   ##
-  # Returns Hash containing the default options for the daemon.
-  def default_options
-    { dir_mode: :normal, dir: File.expand_path('daemon', project.path),
-      multiple: false, log_output: true }
+  # Name of the daemon
+  def daemon_name
+    "MiGA:#{project.name}"
   end
 
   ##
-  # Launches the +task+ with options +opts+ (as command-line arguments) and
-  # returns the process ID as an Integer. If +wait+ it waits for the process to
-  # complete, immediately returns otherwise.
-  # Supported tasks: start, stop, restart, status.
-  def daemon(task, opts = [], wait = true)
-    MiGA.DEBUG "Daemon.daemon #{task} #{opts}"
-    options = default_options
-    opts.unshift(task.to_s)
-    options[:ARGV] = opts
-    # This additional degree of separation below was introduced so the Daemons
-    # package doesn't kill the parent process in workflows.
-    pid = fork do 
-      Daemons.run_proc("MiGA:#{project.name}", options) { while in_loop; end }
-    end
-    Process.wait(pid) if wait
-    pid
+  # Run only in the first loop
+  def daemon_first_loop
+    say '-----------------------------------'
+    say 'MiGA:%s launched' % project.name
+    say '-----------------------------------'
+    load_status
+    say 'Configuration options:'
+    say @runopts.to_s
   end
 
   ##
-  # Tell the world that you're alive.
-  def declare_alive
-    f = File.open(File.expand_path('daemon/alive', project.path), 'w')
-    f.print Time.now.to_s
-    f.close
+  # Run one loop step. Returns a Boolean indicating if the loop should continue.
+  def daemon_loop
+    project.load
+    check_datasets
+    check_project
+    if shutdown_when_done? and jobs_running.size + jobs_to_run.size == 0
+      say 'Nothing else to do, shutting down.'
+      return false
+    end
+    flush!
+    if loop_i >= 12
+      say 'Probing running jobs'
+      @loop_i = 0
+      purge!
+    end
+    report_status
+    sleep(latency)
+    true
   end
 
   ##
   # Report status in a JSON file.
   def report_status
     MiGA::Json.generate(
-      {jobs_running: @jobs_running, jobs_to_run: @jobs_to_run},
-      File.expand_path('daemon/status.json', project.path))
+      { jobs_running: @jobs_running, jobs_to_run: @jobs_to_run },
+      File.join(daemon_home, 'status.json')
+    )
   end
 
   ##
   # Load the status of a previous instance.
   def load_status
-    f_path = File.expand_path('daemon/status.json', project.path)
+    f_path = File.join(daemon_home, 'status.json')
     return unless File.size? f_path
     say 'Loading previous status in daemon/status.json:'
     status = MiGA::Json.parse(f_path)
@@ -234,53 +238,6 @@ class MiGA::Daemon < MiGA::MiGA
     end
   end
 
-  ##
-  # Run one loop step. Returns a Boolean indicating if the loop should continue.
-  def in_loop
-    declare_alive
-    project.load
-    if loop_i == -1
-      say '-----------------------------------'
-      say 'MiGA:%s launched' % project.name
-      say '-----------------------------------'
-      load_status
-      say 'Configuration options:'
-      say @runopts.to_s
-      @loop_i = 0
-    end
-    @loop_i += 1
-    check_datasets
-    check_project
-    if shutdown_when_done? and jobs_running.size + jobs_to_run.size == 0
-      say 'Nothing else to do, shutting down.'
-      return false
-    end
-    flush!
-    if loop_i == 12
-      say 'Probing running jobs'
-      @loop_i = 0
-      purge!
-    end
-    report_status
-    sleep(latency)
-    true
-  end
-
-  ##
-  # Send a datestamped message to the log.
-  def say(*opts)
-    print "[#{Time.new.inspect}] ", *opts, "\n"
-  end
-
-  ##
-  # Terminates a daemon.
-  def terminate
-    say 'Terminating daemon...'
-    report_status
-    f = File.expand_path('daemon/alive', project.path)
-    File.unlink(f) if File.exist? f
-  end
-
   ##
   # Launch the job described by Hash +job+ to +hostk+-th host
   def launch_job(job, hostk = nil)