ace-eye 0.6.1

data/lib/eye/process/commands.rb

@@ -0,0 +1,280 @@
+module Eye::Process::Commands
+
+  def start_process
+    debug 'start_process command'
+
+    switch :starting
+
+    unless self[:start_command]
+      warn 'no :start_command found, unmonitoring'
+      switch :unmonitoring, Eye::Reason.new(:no_start_command)
+      return :no_start_command
+    end
+
+    result = self[:daemonize] ? daemonize_process : execute_process
+
+    if !result[:error]
+      debug "process <#{self.pid}> started successfully"
+      switch :started
+    else
+      error "process <#{self.pid}> failed to start (#{result[:error].inspect})"
+
+      if process_really_running?
+        warn "killing <#{self.pid}> due to error"
+        send_signal(:KILL)
+        sleep 0.2 # little grace
+      end
+
+      self.pid = nil
+      switch :crashed
+    end
+
+    result
+
+  rescue StateMachine::InvalidTransition, Eye::Process::StateError => e
+    warn "wrong switch '#{e.message}'"
+
+    :state_error
+  end
+
+  def stop_process
+    debug 'stop_process command'
+
+    switch :stopping
+
+    kill_process
+
+    if process_really_running?
+      warn "process <#{self.pid}> was not stopped; try checking your command/signals or tuning the stop_timeout/stop_grace values"
+
+      switch :unmonitoring, Eye::Reason.new(:'not stopped (soft command)')
+      nil
+
+    else
+      switch :stopped
+
+      clear_pid_file if self[:clear_pid] # by default for all
+
+      true
+    end
+
+  rescue StateMachine::InvalidTransition, Eye::Process::StateError => e
+    warn "wrong switch '#{e.message}'"
+    nil
+  end
+
+  def restart_process
+    debug 'restart_process command'
+
+    switch :restarting
+
+    if self[:restart_command]
+      execute_restart_command
+      sleep_grace(:restart_grace)
+      result = check_alive_with_refresh_pid_if_needed
+      switch(result ? :restarted : :crashed)
+    else
+      stop_process
+      start_process
+    end
+
+    true
+
+  rescue StateMachine::InvalidTransition, Eye::Process::StateError => e
+    warn "wrong switch '#{e.message}'"
+    nil
+  end
+
+private
+
+  def kill_process
+    unless self.pid
+      error 'cannot stop a process without a pid'
+      return
+    end
+
+    if self[:stop_command]
+      cmd = prepare_command(self[:stop_command])
+      info "executing: `#{cmd}` with stop_timeout: #{self[:stop_timeout].to_f}s and stop_grace: #{self[:stop_grace].to_f}s"
+      res = execute(cmd, config.merge(:timeout => self[:stop_timeout]))
+
+      if res[:error]
+
+        if res[:error].class == Timeout::Error
+          error "stop_command failed with #{res[:error].inspect}; try tuning the stop_timeout value"
+        else
+          error "stop_command failed with #{res[:error].inspect}"
+        end
+      end
+
+      sleep_grace(:stop_grace)
+
+    elsif self[:stop_signals]
+      info "executing stop_signals #{self[:stop_signals].inspect}"
+      stop_signals = self[:stop_signals].clone
+
+      signal = stop_signals.shift
+      send_signal(signal)
+
+      while stop_signals.present?
+        delay = stop_signals.shift
+        signal = stop_signals.shift
+
+        if wait_for_condition(delay.to_f, 0.3){ !process_really_running? }
+          info 'has terminated'
+          break
+        end
+
+        send_signal(signal) if signal
+      end
+
+      sleep_grace(:stop_grace)
+
+    else # default command
+      debug "executing: `kill -TERM #{self.pid}` with stop_grace: #{self[:stop_grace].to_f}s"
+      send_signal(:TERM)
+
+      sleep_grace(:stop_grace)
+
+      # if process not die here, by default we force kill it
+      if process_really_running?
+        warn "process <#{self.pid}> did not die after TERM, sending KILL"
+        send_signal(:KILL)
+        sleep 0.1 # little grace
+      end
+    end
+  end
+
+  def execute_restart_command
+    unless self.pid
+      error 'cannot restart a process without a pid'
+      return
+    end
+
+    cmd = prepare_command(self[:restart_command])
+    info "executing: `#{cmd}` with restart_timeout: #{self[:restart_timeout].to_f}s and restart_grace: #{self[:restart_grace].to_f}s"
+
+    res = execute(cmd, config.merge(:timeout => self[:restart_timeout]))
+
+    if res[:error]
+
+      if res[:error].class == Timeout::Error
+        error "restart_command failed with #{res[:error].inspect}; try tuning the restart_timeout value"
+      else
+        error "restart_command failed with #{res[:error].inspect}"
+      end
+    end
+
+    res
+  end
+
+  def daemonize_process
+    time_before = Time.now
+    res = Eye::System.daemonize(self[:start_command], config)
+    start_time = Time.now - time_before
+
+    info "daemonizing: `#{self[:start_command]}` with start_grace: #{self[:start_grace].to_f}s, env: '#{environment_string}', <#{res[:pid]}> (in #{self[:working_dir]})"
+
+    if res[:error]
+
+      if res[:error].message == 'Permission denied - open'
+        error "daemonize failed with #{res[:error].inspect}; make sure #{[self[:stdout], self[:stderr]]} are writable"
+      else
+        error "daemonize failed with #{res[:error].inspect}"
+      end
+
+      return {:error => res[:error].inspect}
+    end
+
+    self.pid = res[:pid]
+
+    unless self.pid
+      error 'no pid was returned'
+      return {:error => :empty_pid}
+    end
+
+    sleep_grace(:start_grace)
+
+    unless process_really_running?
+      error "process <#{self.pid}> not found, it may have crashed (#{check_logs_str})"
+      return {:error => :not_really_running}
+    end
+
+    # if we using leaf child stratedy, pid should be used as last child process
+    if self[:use_leaf_child]
+      if lpid = Eye::SystemResources.leaf_child(self.pid)
+        info "leaf child for <#{self.pid}> found: <#{lpid}>, accepting it!"
+        self.parent_pid = self.pid
+        self.pid = lpid
+      else
+        warn "leaf child not found for <#{self.pid}>, skipping it"
+      end
+    end
+
+    if control_pid? && !failsafe_save_pid
+      return {:error => :cant_write_pid}
+    end
+
+    res
+  end
+
+  def execute_process
+    info "executing: `#{self[:start_command]}` with start_timeout: #{config[:start_timeout].to_f}s, start_grace: #{self[:start_grace].to_f}s, env: '#{environment_string}' (in #{self[:working_dir]})"
+    time_before = Time.now
+
+    res = execute(self[:start_command], config.merge(:timeout => config[:start_timeout]))
+    start_time = Time.now - time_before
+
+    if res[:error]
+
+      if res[:error].message == 'Permission denied - open'
+        error "execution failed with #{res[:error].inspect}; ensure that #{[self[:stdout], self[:stderr]]} are writable"
+      elsif res[:error].class == Timeout::Error
+        error "execution failed with #{res[:error].inspect}; try increasing the start_timeout value (the current value of #{self[:start_timeout]}s seems too short)"
+      else
+        error "execution failed with #{res[:error].inspect}"
+      end
+
+      return {:error => res[:error].inspect}
+    end
+
+    sleep_grace(:start_grace)
+
+    unless set_pid_from_file
+      error "exit status #{res[:exitstatus]}, pid_file (#{self[:pid_file_ex]}) did not appear within the start_grace period (#{self[:start_grace].to_f}s); check your start_command, or tune the start_grace value (eye expect process to create pid_file in self-daemonization mode)"
+      return {:error => :pid_not_found}
+    end
+
+    unless process_really_running?
+      error "exit status #{res[:exitstatus]}, process <#{self.pid}> (from #{self[:pid_file_ex]}) was not found; ensure that the pid_file is being updated correctly (#{check_logs_str})"
+      return {:error => :not_really_running}
+    end
+
+    res[:pid] = self.pid
+    info "exit status #{res[:exitstatus]}, process <#{res[:pid]}> (from #{self[:pid_file_ex]}) was found"
+    res
+  end
+
+  def check_logs_str
+    if !self[:stdout] && !self[:stderr]
+      'you may want to configure stdout/err/all logs for this process'
+    else
+      "you should check the process logs #{[self[:stdout], self[:stderr]]}"
+    end
+  end
+
+  def prepare_command(command)
+    if self.pid
+      command.to_s.gsub('{{PID}}', self.pid.to_s).gsub('{PID}', self.pid.to_s)
+    else
+      command
+    end
+  end
+
+  def sleep_grace(grace_name)
+    grace = self[grace_name].to_f
+    info "sleeping for :#{grace_name} #{grace}"
+    sleep grace
+  end
+
+end

data/lib/eye/process/config.rb

@@ -0,0 +1,81 @@
+module Eye::Process::Config
+
+  DEFAULTS = {
+    :keep_alive => true, # restart when crashed
+    :check_alive_period => 5.seconds,
+
+    :start_timeout => 15.seconds,
+    :stop_timeout => 10.seconds,
+    :restart_timeout => 10.seconds,
+
+    :start_grace => 2.5.seconds,
+    :stop_grace => 0.5.seconds,
+    :restart_grace => 1.second,
+
+    :daemonize => false,
+    :auto_start => true, # auto start on monitor action
+
+    :children_update_period => 30.seconds,
+    :clear_pid => true, # by default clear pid on stop
+
+    :auto_update_pidfile_grace => 30.seconds,
+    :revert_fuckup_pidfile_grace => 120.seconds,
+  }
+
+  def prepare_config(new_config)
+    h = DEFAULTS.merge(new_config)
+    h[:pid_file_ex] = Eye::System.normalized_file(h[:pid_file], h[:working_dir]) if h[:pid_file]
+    h[:checks] = {} if h[:checks].blank?
+    h[:triggers] = {} if h[:triggers].blank?
+    h[:children_update_period] = h[:monitor_children][:children_update_period] if h[:monitor_children] && h[:monitor_children][:children_update_period]
+
+    # check speedy flapping by default
+    if h[:triggers].blank? || !h[:triggers][:flapping]
+      h[:triggers] ||= {}
+      h[:triggers][:flapping] = {:type => :flapping, :times => 10, :within => 10.seconds}
+    end
+
+    h[:stdout] = Eye::System.normalized_file(h[:stdout], h[:working_dir]) if h[:stdout]
+    h[:stderr] = Eye::System.normalized_file(h[:stderr], h[:working_dir]) if h[:stderr]
+    h[:stdall] = Eye::System.normalized_file(h[:stdall], h[:working_dir]) if h[:stdall]
+
+    h[:environment] = Eye::System.prepare_env(h)
+
+    h
+  end
+
+  def c(name)
+    @config[name]
+  end
+
+  def [](name)
+    @config[name]
+  end
+
+  def update_config(new_config = {})
+    new_config = prepare_config(new_config)
+    @config = new_config
+    @full_name = nil
+    @logger = nil
+
+    debug "updating config to: #{@config.inspect}"
+
+    remove_triggers
+    add_triggers
+
+    if up?
+      # rebuild checks for this process
+      remove_watchers
+      remove_children
+
+      add_watchers
+      add_children
+    end
+  end
+
+  # is pid_file under Eye::Process control, or not
+  def control_pid?
+    !!self[:daemonize]
+  end
+
+end

data/lib/eye/process/controller.rb

@@ -0,0 +1,73 @@
+module Eye::Process::Controller
+
+  def send_command(command, *args)
+    schedule command, *args, Eye::Reason::User.new(command)
+  end
+
+  def start
+    res = if set_pid_from_file
+      if process_really_running?
+        info "process <#{self.pid}> from pid_file is already running"
+        switch :already_running
+        :ok
+      else
+        info "pid_file found, but process <#{self.pid}> is down, starting..."
+        start_process
+      end
+    else
+      info 'pid_file not found, starting...'
+      start_process
+    end
+
+    res
+  end
+
+  def stop
+    stop_process
+    switch :unmonitoring
+  end
+
+  def restart
+    unless pid # unmonitored case
+      try_update_pid_from_file
+    end
+
+    restart_process
+  end
+
+  def monitor
+    if self[:auto_start]
+      start
+    else
+      if try_update_pid_from_file
+        info "process <#{self.pid}> from pid_file is already running"
+        switch :already_running
+      else
+        warn 'process not found, unmonitoring'
+        schedule :unmonitor, Eye::Reason.new(:'not found')
+      end
+    end
+  end
+
+  def unmonitor
+    switch :unmonitoring
+  end
+
+  def delete
+    if self[:stop_on_delete]
+      info 'process has stop_on_delete option, so sync-stop it first'
+      stop
+    end
+
+    remove_watchers
+    remove_children
+    remove_triggers
+
+    terminate
+  end
+
+  def signal(sig = 0)
+    send_signal(sig) if self.pid
+  end
+
+end

data/lib/eye/process/data.rb

@@ -0,0 +1,78 @@
+module Eye::Process::Data
+
+  def logger_tag
+    full_name
+  end
+
+  def app_name
+    self[:application]
+  end
+
+  def group_name
+    (self[:group] == '__default__') ? nil : self[:group]
+  end
+
+  def group_name_pure
+    self[:group]
+  end
+
+  def full_name
+    @full_name ||= [app_name, group_name, self[:name]].compact.join(':')
+  end
+
+  def status_data(debug = false)
+    p_st = self_status_data(debug)
+
+    if children.present?
+      p_st.merge(:subtree => Eye::Utils::AliveArray.new(children.values).map{|c| c.status_data(debug) } )
+    elsif self[:monitor_children] && self.up?
+      p_st.merge(:subtree => [{name: '=loading children='}])
+    else
+      # common state
+      p_st
+    end
+  end
+
+  def self_status_data(debug = false)
+    h = { name: name, state: state,
+          type: (self.class == Eye::ChildProcess ? :child_process : :process),
+          resources: Eye::SystemResources.resources(pid) }
+
+    if @states_history
+      h.merge!( state_changed_at: @states_history.last_state_changed_at.to_i,
+                state_reason: @states_history.last_reason )
+    end
+
+    h.merge!(debug: debug_data) if debug
+    h.merge!(current_command: current_scheduled_command) if current_scheduled_command
+
+    h
+  end
+
+  def debug_data
+    { queue: scheduler_actions_list, watchers: @watchers.keys }
+  end
+
+  def sub_object?(obj)
+    return false if self.class == Eye::ChildProcess
+    self.children.each { |_, child| return true if child == obj }
+    false
+  end
+
+  def environment_string
+    s = []
+    @config[:environment].each { |k, v| s << "#{k}=#{v}" }
+    s * ' '
+  end
+
+  def shell_string(dir = true)
+    str = ''
+    str += "cd #{self[:working_dir]} && " if dir
+    str += environment_string
+    str += ' '
+    str += self[:start_command]
+    str += ' &' if self[:daemonize]
+    str
+  end
+
+end